[MLOP-639] Track logs in S3 (#306)

* Apply tracking logs and logging config.

* Adjusts in CLI and logging.conf.

* Some adjusts.

* Change version to generate new dev package

* Fix version.

* Apply style.

* Add new assert in the migrate unit test.

Author: moromimay

butterfree/__init__.py

@@ -1 +1,8 @@
 """Module docstring example, following Google's docstring style."""
+import logging.config
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("."))
+
+logging.config.fileConfig(fname="butterfree/logging.conf")

butterfree/_cli/__init__.py

@@ -1,10 +0,0 @@
-import logging
-
-
-def __logger(name: str) -> logging.Logger:
-    format_ = "%(name)s:%(asctime)-15s:%(levelname)s:< %(message)s >"
-    logging.basicConfig(format=format_, level=logging.INFO)
-    return logging.getLogger(name)
-
-
-cli_logger = __logger("butterfree")

butterfree/_cli/migrate.py

@@ -115,6 +115,7 @@ class Migrate:
 
     def __init__(self, pipelines: Set[FeatureSetPipeline],) -> None:
         self.pipelines = pipelines
+        self.spark_client = spark_client or SparkClient()
 
     def _send_logs_to_s3(self, file_local: bool, debug_mode: bool) -> None:
         """Send all migration logs to S3."""
@@ -168,6 +169,7 @@ def run(self, generate_logs: bool = False, debug_mode: bool = False) -> None:
 
         self._send_logs_to_s3(generate_logs, debug_mode)
 
+        self._send_logs_to_s3(generate_logs)
 
 @app.command("apply")
 def migrate(

butterfree/logging.conf

@@ -0,0 +1,52 @@
+[loggers]
+keys=root,cli,migrate,database_migrate
+
+[handlers]
+keys=consoleHandler,file
+
+[formatters]
+keys=simpleFormatter,jsonFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_cli]
+level=DEBUG
+handlers=file
+qualname=cli
+propagate=0
+
+[logger_migrate]
+level=DEBUG
+handlers=file
+qualname=migrate
+propagate=0
+
+[logger_database_migrate]
+level=DEBUG
+handlers=file
+qualname=database_migrate
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=simpleFormatter
+args=(sys.stdout,)
+
+[handler_file]
+class=FileHandler
+level=DEBUG
+formatter=jsonFormatter
+args=('logs/logging.json', "a")
+
+[formatter_simpleFormatter]
+format=%(name)s:%(asctime)-15s:%(levelname)s:%(message)s
+datefmt=
+class=logging.Formatter
+
+[formatter_jsonFormatter]
+format={"name": "%(name)s", "timestamp": "%(asctime)-15s", "level": "%(levelname)s", "message": "%(message)s"}
+datefmt=
+class=logging.Formatter

logs/logging.json

@@ -9,6 +9,10 @@
 
 runner = CliRunner()
 
+    def test_migrate_all_pairs(self, mocker):
+        mocker.patch.object(MetastoreMigration, "apply_migration")
+        mocker.patch.object(CassandraMigration, "apply_migration")
+        mocker.patch.object(migrate.Migrate, "_send_logs_to_s3")
 
 class TestMigrate:
     def test_migrate_success(self, mocker):