Allow user to change the logger server port. This fixes the issue #2 where user app did not exit cleanly at previous run

Author: Dragon2fly

README.md

@@ -127,11 +127,13 @@ setup_logging(config_path="", log_path="",
               use_multiprocessing=False,
               limit_line_length=1000, 
               analyze_raise_statement=False,
+              host="",
+              port=0,
               )
 ```
 
 This function also return a `LogConfig` object. 
-Except `config_path`, `log_path` and `use_multiprocessing`, 
+Except `config_path`, `log_path`, `use_multiprocessing`, `host` and `port`, 
 other parameters are attributes of this object and can be changed on the fly.
 
 Except `config_path`, `log_path`, all other parameters can be defined in `logger_tt` section in the config file
@@ -507,6 +509,18 @@ The content of `log.txt` should be similar to below:
    **Note**: Under linux, to use `queueHandler`, you must pass `use_multiprocessing="fork"` to `setup_logging`.<br>
    Other options `True`, `spawn`, `forkserver` will use `socketHandler` by default.<br> 
    This is to prevent you `set_start_method` as `spawn` under linux and thus `queueHandler` won't work.
+
+
+   **Socket Address**: `socketHandler` will use tcp `localhost` and port `9020` by default. 
+   In the rare cases where you run multiple multiprocessing applications with `logger_tt`, 
+   the `Address already in use` error will be raised. In such cases, you have to set the address manually.
+
+```python
+setup_logging(host='localhost', port=6789)
+```
+   You can omit the `host` if you use `"localhost"`. 
+   You can also set this in the log config file for each application. 
+
 
 ### 8. Temporary disable logging:
 
@@ -875,10 +889,14 @@ logger_tt:
 ## 1.7.0
 * Fixed: 
   * multiprocessing: log file rollover fails as child process keep opening the file.
-  * multiprocessing: if the log path is set by variable with time, 
+  * multiprocessing: if the log path is set by a variable with time, 
     child process creates a new redundant log path.
-* New functionality: Added `StreamHandlerWithBuffer`. 
-    GUI app could use this handler to keep the app responsive while having a tremendous log output. 
+  
+* New functionality: Added `StreamHandlerWithBuffer`. Buffer the log output by time or by line number.
+    GUI app could use this handler to keep the app responsive while having a tremendous log output.
+
+* Usability: In multiprocessing logging, 
+  users can set the log server address themselves through `setup_logging` or log config file.
 
 ## 1.6.1
 * Added `limit_line_length` parameter: log only maximum `n` characters for each traceback line. 
@@ -933,7 +951,7 @@ logger_tt:
 **Pre-existing loggers:**<br> 
 Before this version, if you import submodules before importing `logger_tt` and 
 there are loggers in submodules, these loggers do not inspect exception when you call `logger.exception()`. 
-That is because there class was different than loggers created after importing `logger_tt`.
+That is because there class was different from the loggers created after importing `logger_tt`.
 Now all loggers have the same new class regardless the point of importing `logger_tt`.
 
 **setup_logging()**: This function should only be called once. 

logger_tt/__init__.py

@@ -111,6 +111,7 @@ def merge_config(from_file: dict, from_func: dict) -> dict:
                     full_context=0, suppress=None,
                     suppress_level_below=logging.WARNING, use_multiprocessing=False,
                     limit_line_length=1000, analyze_raise_statement=False,
+                    host=None, port=None,
                     )
     merged = {}
     for key, val in defaults.items():
@@ -149,6 +150,8 @@ def setup_logging(config_path: str = "", log_path: str = "", **logger_tt_config)
                                     queue.Queue to multiprocessing.Queue . This option can only be used here.
         :key limit_line_length   : int, define how long should one log line be. 0: unlimited; n: n character
         :key analyze_raise_statement: bool, should the variables in `raise` exception line be shown or not.
+        :key host: str, default to 'localhost'. Used in multiprocessing logging
+        :key port: int, default to logging.handlers.DEFAULT_TCP_LOGGING_PORT. Used in multiprocessing logging
     """
     if internal_config.initialized:
         logger.warning('Re-initializing logger_tt. "setup_logging()" should only be called one.')

logger_tt/__init__.pyi

@@ -21,4 +21,6 @@ def setup_logging(config_path: str = "", log_path: str = "",
                   suppress_level_below: int = logging.WARNING,
                   use_multiprocessing: bool = False,
                   limit_line_length: int = 1000,
-                  analyze_raise_statement: bool = False) -> LogConfig: ...
+                  analyze_raise_statement: bool = False,
+                  host: str = None,
+                  port: int = None) -> LogConfig: ...

logger_tt/core.py

@@ -26,8 +26,8 @@ def __init__(self):
         self.q_listeners = []
 
         # tcp port for socket handler
-        self.host = 'localhost'
-        self.port = handlers.DEFAULT_TCP_LOGGING_PORT
+        self._host = 'localhost'
+        self._port = handlers.DEFAULT_TCP_LOGGING_PORT
         self.tcp_server = None
 
         # other settings
@@ -74,6 +74,10 @@ def from_dict(self, odict: dict):
         # backup root handler:
         self.root_handlers = root_logger.handlers
 
+        # host and port for multiprocessing logging
+        self._host = odict.get('host') or 'localhost'
+        self._port = odict.get('port') or handlers.DEFAULT_TCP_LOGGING_PORT
+
         # set logging mode accordingly
         self._set_mode(odict['use_multiprocessing'])
 
@@ -137,13 +141,13 @@ def _replace_with_socket_handler(self):
         logger.handlers = []
 
         # add socket handler
-        socket_handler = logging.handlers.SocketHandler(self.host, self.port)
+        socket_handler = logging.handlers.SocketHandler(self._host, self._port)
         atexit.register(socket_handler.close)
         logger.addHandler(socket_handler)
 
         # initiate server
         if current_process().name == 'MainProcess':
-            self.tcp_server = LogRecordSocketReceiver(self.host, self.port, all_handlers)
+            self.tcp_server = LogRecordSocketReceiver(self._host, self._port, all_handlers)
             serving = Thread(target=self.tcp_server.serve_until_stopped)
             serving.start()
             root_logger.debug('Logging server started!')

logger_tt/log_config.json

@@ -65,6 +65,8 @@
     "use_multiprocessing": false,
     "limit_line_length": 1000,
     "analyze_raise_statement": false,
+    "host": "",
+    "port": 0,
     "default_logger_formats": {
       "normal": ["%(name)s", "%(filename)s"],
       "thread": ["%(message)s", "%(threadName)s %(message)s"],

logger_tt/log_config.yaml

@@ -29,12 +29,12 @@ handlers:
     delay: True
 
   buffer_stream_handler:
-    class: logger_tt.handlers.StreamHandlerWithBuffer,
-    level: DEBUG,
-    formatter: brief,
-    stream: ext://sys.stdout,
-    buffer_time: 0.5,
-    buffer_lines: 0,
+    class: logger_tt.handlers.StreamHandlerWithBuffer
+    level: DEBUG
+    formatter: brief
+    stream: ext://sys.stdout
+    buffer_time: 0.5
+    buffer_lines: 0
     debug: False
 
 loggers:
@@ -57,6 +57,8 @@ logger_tt:
   use_multiprocessing: False
   limit_line_length: 1000
   analyze_raise_statement: False
+  host: ""
+  port: 0
   default_logger_formats:
     normal: ["%(name)s", "%(filename)s"]
     thread: ["%(message)s", "%(threadName)s %(message)s"]

tests/multiprocessing_change_port.py

@@ -0,0 +1,39 @@
+import time
+import sys
+from random import randint
+
+from multiprocessing import Process
+from logger_tt import setup_logging
+from logging import getLogger
+
+
+__author__ = "Duc Tin"
+logger = getLogger(__name__)
+config = setup_logging(config_path="multiprocessing_change_port.yaml")
+
+
+def worker(arg):
+    logger.info(f'child process {arg}: started')
+    time.sleep(randint(3, 10))
+    logger.info(f'child process {arg}: stopped')
+
+
+if __name__ == '__main__':
+    if len(sys.argv) > 1:
+        proc_no = int(sys.argv[1])
+    else:
+        proc_no = 7
+
+    all_processes = []
+    logger.info('Parent process is ready to spawn child')
+    logger.info(f'current logging tcp_server is: {config.tcp_server.server_address}')
+    for i in range(proc_no):
+        p = Process(target=worker, args=(i,))
+        all_processes.append(p)
+        p.daemon = True
+        p.start()
+
+    for p in all_processes:
+        p.join()
+
+    print('__finished__')

tests/test_multiprocessing.py

@@ -100,3 +100,24 @@ def test_multiprocessing_issue5():
     logfile = sub_folders[0] / 'info.log'
     assert logfile.exists()
     assert 'Test for issue 5' in logfile.read_text()
+
+
+def test_multiprocessing_port_change():
+    """Change the tcp server's port to a user selected one"""
+    yaml = YAML(typ='safe')
+    config_file = Path("../logger_tt/log_config.yaml")
+    log_config = yaml.load(config_file.read_text())
+    log_config['logger_tt']['use_multiprocessing'] = True
+    log_config['logger_tt']['port'] = 6789
+
+    # write the config out
+    test_config = Path("multiprocessing_change_port.yaml")
+    yaml.dump(data=log_config, stream=test_config)
+
+    cmd = [sys.executable, "multiprocessing_change_port.py", "3"]
+    result = run(cmd, stdout=PIPE, universal_newlines=True)
+
+    test_config.unlink()
+    assert '6789' in result.stdout, "Port failed to change"
+    assert result.stdout.count("stopped") == 3, "Child process failed to log"
+