fix: publish and dev versions (#359)

* fix: publish, versions, tests

Author: ralphrass

.github/workflows/publish.yml

@@ -9,8 +9,7 @@ jobs:
   Pipeline:
     if: github.ref == 'refs/heads/master'
 
-    runs-on: ubuntu-22.04
-    container: quintoandar/python-3-7-java
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2

Makefile

@@ -9,8 +9,8 @@ VERSION := $(shell grep __version__ setup.py | head -1 | cut -d \" -f2 | cut -d
 .PHONY: environment
 ## create virtual environment for butterfree
 environment:
-	@pyenv install -s 3.7.13
-	@pyenv virtualenv 3.7.13 butterfree
+	@pyenv install -s 3.9.19
+	@pyenv virtualenv 3.9.19 butterfree
 	@pyenv local butterfree
 	@PYTHONPATH=. python -m pip install --upgrade pip
 

butterfree/_cli/main.py

@@ -2,7 +2,7 @@
 
 from butterfree._cli import migrate
 
-app = typer.Typer()
+app = typer.Typer(no_args_is_help=True)
 app.add_typer(migrate.app, name="migrate")
 
 if __name__ == "__main__":

butterfree/_cli/migrate.py

@@ -16,7 +16,9 @@
 from butterfree.migrations.database_migration import ALLOWED_DATABASE
 from butterfree.pipelines import FeatureSetPipeline
 
-app = typer.Typer(help="Apply the automatic migrations in a database.")
+app = typer.Typer(
+    help="Apply the automatic migrations in a database.", no_args_is_help=True
+)
 
 logger = __logger("migrate", True)
 
@@ -89,7 +91,7 @@ def __fs_objects(path: str) -> Set[FeatureSetPipeline]:
             instances.add(value)
 
     logger.info("Creating instances...")
-    return set(value() for value in instances)
+    return set(value() for value in instances)  # type: ignore
 
 
 PATH = typer.Argument(

butterfree/clients/__init__.py

@@ -1,4 +1,5 @@
 """Holds connection clients."""
+
 from butterfree.clients.abstract_client import AbstractClient
 from butterfree.clients.cassandra_client import CassandraClient
 from butterfree.clients.spark_client import SparkClient

butterfree/clients/abstract_client.py

@@ -1,6 +1,7 @@
 """Abstract class for database clients."""
+
 from abc import ABC, abstractmethod
-from typing import Any
+from typing import Any, Optional
 
 
 class AbstractClient(ABC):
@@ -25,7 +26,7 @@ def sql(self, query: str) -> Any:
         pass
 
     @abstractmethod
-    def get_schema(self, table: str, database: str = None) -> Any:
+    def get_schema(self, table: str, database: Optional[str] = None) -> Any:
         """Returns desired table schema.
 
         Attributes:

butterfree/clients/cassandra_client.py

@@ -1,4 +1,5 @@
 """CassandraClient entity."""
+
 from ssl import CERT_REQUIRED, PROTOCOL_TLSv1
 from typing import Dict, List, Optional
 
@@ -102,7 +103,9 @@ def sql(self, query: str) -> ResponseFuture:
         """
         return self.conn.execute(query)
 
-    def get_schema(self, table: str, database: str = None) -> List[Dict[str, str]]:
+    def get_schema(
+        self, table: str, database: Optional[str] = None
+    ) -> List[Dict[str, str]]:
         """Returns desired table schema.
 
         Attributes:

butterfree/clients/spark_client.py

@@ -69,7 +69,7 @@ def read(
 
         return df_reader.format(format).load(path=path, **options)  # type: ignore
 
-    def read_table(self, table: str, database: str = None) -> DataFrame:
+    def read_table(self, table: str, database: Optional[str] = None) -> DataFrame:
         """Use the SparkSession.read interface to read a metastore table.
 
         Args:
@@ -179,9 +179,9 @@ def write_table(
         database: Optional[str],
         table_name: str,
         path: str,
-        format_: str = None,
-        mode: str = None,
-        partition_by: List[str] = None,
+        format_: Optional[str] = None,
+        mode: Optional[str] = None,
+        partition_by: Optional[List[str]] = None,
         **options: Any,
     ) -> None:
         """Receive a spark DataFrame and write it as a table in metastore.
@@ -231,7 +231,10 @@ def create_temporary_view(dataframe: DataFrame, name: str) -> Any:
         return dataframe.writeStream.format("memory").queryName(name).start()
 
     def add_table_partitions(
-        self, partitions: List[Dict[str, Any]], table: str, database: str = None
+        self,
+        partitions: List[Dict[str, Any]],
+        table: str,
+        database: Optional[str] = None,
     ) -> None:
         """Add partitions to an existing table.
 
@@ -259,9 +262,11 @@ def add_table_partitions(
         key_values_expr = [
             ", ".join(
                 [
-                    "{} = {}".format(k, v)
-                    if not isinstance(v, str)
-                    else "{} = '{}'".format(k, v)
+                    (
+                        "{} = {}".format(k, v)
+                        if not isinstance(v, str)
+                        else "{} = '{}'".format(k, v)
+                    )
                     for k, v in partition.items()
                 ]
             )
@@ -314,7 +319,9 @@ def _convert_schema(self, schema: DataFrame) -> List[Dict[str, str]]:
 
         return converted_schema
 
-    def get_schema(self, table: str, database: str = None) -> List[Dict[str, str]]:
+    def get_schema(
+        self, table: str, database: Optional[str] = None
+    ) -> List[Dict[str, str]]:
         """Returns desired table schema.
 
         Attributes:

butterfree/configs/db/cassandra_config.py

@@ -1,4 +1,5 @@
 """Holds configurations to read and write with Spark to Cassandra DB."""
+
 from typing import Any, Dict, List, Optional
 
 from butterfree.configs import environment
@@ -32,18 +33,18 @@ class CassandraConfig(AbstractWriteConfig):
 
     def __init__(
         self,
-        username: str = None,
-        password: str = None,
-        host: str = None,
-        keyspace: str = None,
-        mode: str = None,
-        format_: str = None,
-        stream_processing_time: str = None,
-        stream_output_mode: str = None,
-        stream_checkpoint_path: str = None,
-        read_consistency_level: str = None,
-        write_consistency_level: str = None,
-        local_dc: str = None,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        host: Optional[str] = None,
+        keyspace: Optional[str] = None,
+        mode: Optional[str] = None,
+        format_: Optional[str] = None,
+        stream_processing_time: Optional[str] = None,
+        stream_output_mode: Optional[str] = None,
+        stream_checkpoint_path: Optional[str] = None,
+        read_consistency_level: Optional[str] = None,
+        write_consistency_level: Optional[str] = None,
+        local_dc: Optional[str] = None,
     ):
         self.username = username
         self.password = password

butterfree/configs/db/kafka_config.py

@@ -1,4 +1,5 @@
 """Holds configurations to read and write with Spark to Kafka."""
+
 from typing import Any, Dict, List, Optional
 
 from butterfree.configs import environment
@@ -25,13 +26,13 @@ class KafkaConfig(AbstractWriteConfig):
 
     def __init__(
         self,
-        kafka_topic: str = None,
-        kafka_connection_string: str = None,
-        mode: str = None,
-        format_: str = None,
-        stream_processing_time: str = None,
-        stream_output_mode: str = None,
-        stream_checkpoint_path: str = None,
+        kafka_topic: Optional[str] = None,
+        kafka_connection_string: Optional[str] = None,
+        mode: Optional[str] = None,
+        format_: Optional[str] = None,
+        stream_processing_time: Optional[str] = None,
+        stream_output_mode: Optional[str] = None,
+        stream_checkpoint_path: Optional[str] = None,
     ):
         self.kafka_topic = kafka_topic
         self.kafka_connection_string = kafka_connection_string
@@ -147,4 +148,4 @@ def translate(self, schema: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
             Kafka schema.
 
         """
-        pass
+        return [{}]