Merge pull request #2 from adybbroe/add-logging

Add support for logging

Author: mraspaud

pytroll_runner/__init__.py

@@ -20,6 +20,8 @@
 
 """
 import argparse
+import logging
+import logging.config
 import os
 import re
 from contextlib import closing
@@ -31,25 +33,46 @@
 from posttroll.publisher import create_publisher_from_dict_config
 from posttroll.subscriber import create_subscriber_from_dict_config
 
+logger = logging.getLogger('pytroll-runner')
+
 
 def main(args=None):
     """Main script."""
     parsed_args = parse_args(args=args)
+    setup_logging(parsed_args.log_config)
+
+    logger.info("Start generic pytroll-runner")
     return run_and_publish(parsed_args.config_file)
 
 
+def setup_logging(config_file):
+    """Setup the logging from a log config yaml file."""
+    if config_file is not None:
+        with open(config_file) as fd:
+            log_dict = yaml.safe_load(fd.read())
+            logging.config.dictConfig(log_dict)
+            return
+
+
 def parse_args(args=None):
-    """Parse commandline arguments."""
+    """Parse command line arguments."""
     parser = argparse.ArgumentParser("Pytroll Runner",
                                      description="Automate third party software in a pytroll environment")
     parser.add_argument("config_file",
                         help="The configuration file to run on.")
+    parser.add_argument("-l", "--log_config",
+                        help="The log configuration yaml file.",
+                        default=None)
     return parser.parse_args(args)
 
 
 def run_and_publish(config_file):
     """Run the command and publish the expected files."""
     command_to_call, subscriber_config, publisher_config = read_config(config_file)
+    logger.debug("Subscriber config settings: ")
+    for item in subscriber_config:
+        logger.debug(f"{item} = {str(subscriber_config[item])}")
+
     preexisting_files = check_existing_files(publisher_config)
 
     with closing(create_publisher_from_dict_config(publisher_config["publisher_settings"])) as pub:
@@ -84,6 +107,7 @@ def read_config(config_file):
 
 def run_from_new_subscriber(command, subscriber_settings):
     """Run the command with files gotten from a new subscriber."""
+    logger.debug("Run from new subscriber...")
     with closing(create_subscriber_from_dict_config(subscriber_settings)) as sub:
         return run_on_messages(command, sub.recv())
 
@@ -106,6 +130,7 @@ def run_on_files(command, files):
     """Run the command of files."""
     if not files:
         return
+    logger.info(f"Start running command {command} on files {files}")
     process = Popen([os.fspath(command), *files], stdout=PIPE)
     out, _ = process.communicate()
     return out

pytroll_runner/tests/test_runner.py

@@ -1,5 +1,6 @@
 """Tests for the pytroll runner."""
 import os
+import logging
 from unittest import mock
 
 import pytest
@@ -45,6 +46,29 @@
 """
 
 
+log_config_content = """version: 1
+disable_existing_loggers: false
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    stream: ext://sys.stdout
+root:
+  level: DEBUG
+  handlers: [console]
+"""
+
+
+@pytest.fixture()
+def log_config_file(tmp_path):
+    """Write a log config file."""
+    log_config = tmp_path / "mylogconfig.yaml"
+    with open(log_config, "w") as fobj:
+        fobj.write(log_config_content)
+
+    return log_config
+
+
 @pytest.fixture()
 def command(tmp_path):
     """Make a command script that just prints out the files it got."""
@@ -266,6 +290,7 @@ def single_file_to_glob(tmp_path):
     pattern = os.fspath(tmp_path / "file?")
     return pattern
 
+
 @pytest.fixture()
 def old_generated_file(tmp_path):
     """Create a single file to glob."""
@@ -276,7 +301,7 @@ def old_generated_file(tmp_path):
     return filename
 
 
-def test_run_and_publish(tmp_path, command_bla, files_to_glob):
+def test_run_and_publish(caplog, tmp_path, command_bla, files_to_glob):
     """Test run and publish."""
     sub_config = dict(nameserver=False, addresses=["ipc://bla"])
     pub_config = dict(publisher_settings=dict(nameservers=False, port=1979),
@@ -294,12 +319,17 @@ def test_run_and_publish(tmp_path, command_bla, files_to_glob):
     data = {"dataset": [{"uri": os.fspath(tmp_path / f), "uid": f} for f in some_files]}
     messages = [Message("some_topic", "dataset", data=data)]
 
-    with patched_subscriber_recv(messages):
-        with patched_publisher() as published_messages:
-            run_and_publish(yaml_file)
-            assert len(published_messages) == 1
-            for ds, filename in zip(published_messages[0].data["dataset"], some_files):
-                assert ds["uid"] == filename + ".bla"
+    with caplog.at_level(logging.DEBUG):
+        with patched_subscriber_recv(messages):
+            with patched_publisher() as published_messages:
+                run_and_publish(yaml_file)
+                assert len(published_messages) == 1
+                for ds, filename in zip(published_messages[0].data["dataset"], some_files):
+                    assert ds["uid"] == filename + ".bla"
+
+    assert "Subscriber config settings: " in caplog.text
+    assert "addresses = ['ipc://bla']" in caplog.text
+    assert "nameserver = False" in caplog.text
 
 
 def test_run_and_publish_does_not_pick_old_files(tmp_path, command_bla, files_to_glob, old_generated_file):
@@ -370,7 +400,6 @@ def test_run_and_publish_with_files_from_log(tmp_path, command_aws):
             assert published_messages[0].data["uri"] == "/local_disk/aws_test/test/RAD_AWS_1B/" + expected
 
 
-
 def test_config_reader(command, tmp_path):
     """Test the config reader."""
     sub_config = dict(nameserver=False, addresses=["ipc://bla"])
@@ -383,6 +412,7 @@ def test_config_reader(command, tmp_path):
     with open(yaml_file, "w") as fd:
         fd.write(yaml.dump(test_config))
     command_to_call, subscriber_config, publisher_config = read_config(yaml_file)
+
     assert subscriber_config == sub_config
     assert command_to_call == command_path
     assert publisher_config == pub_config
@@ -398,3 +428,27 @@ def test_main_crashes_when_config_file_missing():
     """Test that main crashes when the config file is missing."""
     with pytest.raises(FileNotFoundError):
         main(["moose_config.yaml"])
+
+
+def test_main_parse_log_configfile(tmp_path, command, log_config_file):
+    """Test that when parsing a log-config yaml file logging is being setup"""
+    sub_config = dict(nameserver=False, addresses=["ipc://bla"])
+    pub_settings = dict(nameserver=False, name='blabla')
+    pub_config = dict(topic="/hi/there/",
+                      expected_files='*.bufr',
+                      publisher_settings=pub_settings)
+    command_path = os.fspath(command)
+    test_config = dict(subscriber_config=sub_config,
+                       script=command_path,
+                       publisher_config=pub_config)
+
+    yaml_file = tmp_path / "config.yaml"
+    with open(yaml_file, "w") as fd:
+        fd.write(yaml.dump(test_config))
+
+    messages = []
+    with patched_subscriber_recv(messages):
+        with patched_publisher() as published_messages:
+            main([str(yaml_file), '-l', str(log_config_file)])
+
+    assert isinstance(logging.getLogger('').handlers[0], logging.StreamHandler)