bpo-32146: multiprocessing freeze_support needed outside win32

Doc/library/multiprocessing.rst

@@ -926,8 +926,8 @@ Miscellaneous
 .. function:: freeze_support()
 
    Add support for when a program which uses :mod:`multiprocessing` has been
-   frozen to produce a Windows executable.  (Has been tested with **py2exe**,
-   **PyInstaller** and **cx_Freeze**.)
+   frozen to produce an application executable.  (Has been tested with
+   **py2exe**, **PyInstaller** and **cx_Freeze**.)
 
    One needs to call this function straight after the ``if __name__ ==
    '__main__'`` line of the main module.  For example::
@@ -942,12 +942,17 @@ Miscellaneous
           Process(target=f).start()
 
    If the ``freeze_support()`` line is omitted then trying to run the frozen
-   executable will raise :exc:`RuntimeError`.
+   executable will cause errors (e.g., :exc:`RuntimeError`). It is needed
+   when using the ``'spawn'`` and ``'forkserver'`` start methods.
 
-   Calling ``freeze_support()`` has no effect when invoked on any operating
-   system other than Windows.  In addition, if the module is being run
-   normally by the Python interpreter on Windows (the program has not been
-   frozen), then ``freeze_support()`` has no effect.
+
+   ``freeze_support()`` has no effect when invoked in a module that is being
+   run normally by the Python interpreter (i.e., instead of by a frozen
+   executable).
+
+.. versionchanged:: 3.7
+   Now supported on Unix (for the ``'spawn'`` and ``'forkserver'` start
+   methods)
 
 .. function:: get_all_start_methods()
 

Lib/multiprocessing/context.py

@@ -144,7 +144,7 @@ def freeze_support(self):
         '''Check whether this is a fake forked process in a frozen executable.
         If so then run code specified by commandline and exit.
         '''
-        if sys.platform == 'win32' and getattr(sys, 'frozen', False):
+        if getattr(sys, 'frozen', False):
             from .spawn import freeze_support
             freeze_support()
 

Lib/multiprocessing/forkserver.py

@@ -127,7 +127,10 @@ def ensure_running(self):
                     cmd %= (listener.fileno(), alive_r, self._preload_modules,
                             data)
                     exe = spawn.get_executable()
-                    args = [exe] + util._args_from_interpreter_flags()
+                    args = [exe]
+                    if getattr(sys, 'frozen', False):
+                        args.append('--multiprocessing-forkserver')
+                    args += util._args_from_interpreter_flags()
                     args += ['-c', cmd]
                     pid = util.spawnv_passfds(exe, args, fds_to_pass)
                 except:

Lib/multiprocessing/semaphore_tracker.py

@@ -66,7 +66,10 @@ def ensure_running(self):
                 fds_to_pass.append(r)
                 # process will out live us, so no need to wait on pid
                 exe = spawn.get_executable()
-                args = [exe] + util._args_from_interpreter_flags()
+                args = [exe]
+                if getattr(sys, 'frozen', False):
+                    args.append('--multiprocessing-semaphore-tracker')
+                args += util._args_from_interpreter_flags()
                 args += ['-c', cmd % r]
                 pid = util.spawnv_passfds(exe, args, fds_to_pass)
             except:

Lib/multiprocessing/spawn.py

@@ -8,6 +8,7 @@
 # Licensed to PSF under a Contributor Agreement.
 #
 
+import ast
 import os
 import sys
 import runpy
@@ -49,6 +50,7 @@ def get_executable():
 #
 #
 
+
 def is_forking(argv):
     '''
     Return whether commandline indicates we are forking
@@ -59,19 +61,91 @@ def is_forking(argv):
         return False
 
 
+def get_forking_args(argv):
+    '''
+    If the command line indicated we are forking, return (args, kwargs)
+    suitable for passing to spawn_main. Otherwise return None.
+    '''
+    if not is_forking(argv):
+        return None
+
+    args = []
+    kwds = {}
+    for arg in argv[2:]:
+        name, value = arg.split('=')
+        if value == 'None':
+            kwds[name] = None
+        else:
+            kwds[name] = int(value)
+
+    return args, kwds
+
+
+def get_semaphore_tracker_args(argv):
+    '''
+    If the command line indicates we are running the semaphore tracker,
+    return (args, kwargs) suitable for passing to semaphore_tracker.main.
+    Otherwise return None.
+    '''
+    if len(argv) < 2 or argv[1] != '--multiprocessing-semaphore-tracker':
+        return None
+
+    # command ends with main(fd) - extract fd
+    r = int(argv[-1].rsplit('(')[1].split(')')[0])
+
+    args = [r]
+    kwds = {}
+    return args, kwds
+
+
+def get_forkserver_args(argv):
+    '''
+    If the command line indicates we are running the forkserver, return
+    (args, kwargs) suitable for passing to forkserver.main. Otherwise return
+    None.
+    '''
+    if len(argv) < 2 or argv[1] != '--multiprocessing-forkserver':
+        return None
+
+    # command ends with main(listener_fd, alive_r, preload, **kwds) - extract
+    # the args and kwarfs
+    # listener_fd and alive_r are integers
+    # preload is a list
+    # kwds map strings to lists
+    main_args = argv[-1].split('main(')[1].rsplit(')', 1)[0].split(', ', 3)
+    listener_fd = int(main_args[0])
+    alive_r = int(main_args[1])
+    preload = ast.literal_eval(main_args[2])
+
+    args = [listener_fd, alive_r, preload]
+    kwds = ast.literal_eval(main_args[3][2:])
+    return args, kwds
+
+
 def freeze_support():
     '''
-    Run code for process object if this in not the main process
+    Run code for process object if this in not the main process.
     '''
-    if is_forking(sys.argv):
-        kwds = {}
-        for arg in sys.argv[2:]:
-            name, value = arg.split('=')
-            if value == 'None':
-                kwds[name] = None
-            else:
-                kwds[name] = int(value)
-        spawn_main(**kwds)
+    argv = sys.argv
+
+    forking_args = get_forking_args(argv)
+    if forking_args is not None:
+        args, kwds = forking_args
+        spawn_main(*args, **kwds)
+        sys.exit()
+
+    semaphore_tracker_args = get_semaphore_tracker_args(argv)
+    if semaphore_tracker_args is not None:
+        from multiprocessing.semaphore_tracker import main
+        args, kwds = semaphore_tracker_args
+        main(*args, **kwds)
+        sys.exit()
+
+    forkserver_args = get_forkserver_args(argv)
+    if get_forkserver_args(sys.argv):
+        from multiprocessing.forkserver import main
+        args, kwds = forkserver_args
+        main(*args, **kwds)
         sys.exit()
 
 

Lib/test/_test_multiprocessing.py

@@ -3,6 +3,7 @@
 #
 
 import unittest
+import unittest.mock
 import queue as pyqueue
 import contextlib
 import time
@@ -4493,6 +4494,76 @@ def test_empty(self):
 
         proc.join()
 
+
+#
+# Issue 32146: freeze_support for fork, spawn, and forkserver start methods
+#
+
+class TestFreezeSupport(unittest.TestCase):
+    def setUp(self):
+        import multiprocessing.spawn
+        self.module = multiprocessing.spawn
+
+    def test_get_forking_args(self):
+        # Too few args
+        self.assertIsNone(self.module.get_forking_args(['./embed']))
+
+        # Wrong second argument
+        self.assertIsNone(
+            self.module.get_forking_args(['./embed', '-h'])
+        )
+
+        # All correct
+        args, kwds = self.module.get_forking_args(
+            ['./embed', '--multiprocessing-fork', 'pipe_handle=6', 'key=None']
+        )
+        self.assertEqual(args, [])
+        self.assertEqual(kwds, {'pipe_handle': 6, 'key': None})
+
+    def test_get_semaphore_tracker_args(self):
+        # Too few args
+        self.assertIsNone(self.module.get_semaphore_tracker_args(['.embed']))
+
+        # Wrong second argument
+        self.assertIsNone(self.module.get_semaphore_tracker_args(
+            ['./embed', '-h'])
+        )
+
+        # All correct
+        argv = [
+            './embed',
+            '--multiprocessing-semaphore-tracker',
+            'from multiprocessing.semaphore_tracker import main;main(5)'
+        ]
+        args, kwds = self.module.get_semaphore_tracker_args(argv)
+        self.assertEqual(args, [5])
+        self.assertEqual(kwds, {})
+
+    def test_get_forkserver_args(self):
+        # Too few args
+        self.assertFalse(self.module.get_forkserver_args(['./python-embed']))
+
+        # Wrong second argument
+        self.assertFalse(
+            self.module.get_forkserver_args(['./python-embed', '-h'])
+        )
+
+        # All correct
+        argv = [
+            './embed',
+            '--multiprocessing-forkserver',
+            (
+                "from multiprocessing.forkserver import main; "
+                "main(8, 9, ['__main__'], "
+                "**{'sys_path': ['/embed/lib', '/embed/lib/library.zip']})"
+            )
+        ]
+        args, kwds = self.module.get_forkserver_args(argv)
+        self.assertEqual(args, [8, 9, ['__main__']])
+        self.assertEqual(
+            kwds, {'sys_path': ['/embed/lib', '/embed/lib/library.zip']}
+        )
+
 #
 # Mixins
 #

Misc/NEWS.d/next/Library/2018-01-14-16-34-26.bpo-32146.6pFDbn.rst

@@ -0,0 +1,2 @@
+``multiprocessing.freeze_support`` now works on non-Windows platforms as
+well.