From: Arkadiusz Miśkiewicz Date: Sun, 15 Mar 2020 20:57:06 +0000 (+0100) Subject: - rel 2; fixes some multiprocessing hang - https://bugs.python.org/issue38744 X-Git-Tag: auto/th/python3-3.8.2-2 X-Git-Url: https://git.pld-linux.org/?a=commitdiff_plain;h=refs%2Ftags%2Fauto%2Fth%2Fpython3-3.8.2-2;hp=71237db43410b763ca7e9b3513687e073ff0a2ef;p=packages%2Fpython3.git - rel 2; fixes some multiprocessing hang - https://bugs.python.org/issue38744 --- diff --git a/bug-38744.patch b/bug-38744.patch new file mode 100644 index 0000000..259b547 --- /dev/null +++ b/bug-38744.patch @@ -0,0 +1,90 @@ +From ef8bff34231bb9c74fe349905bc29b5be8a48325 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Batuhan=20Ta=C5=9Fkaya?= + <47358913+isidentical@users.noreply.github.com> +Date: Sun, 15 Mar 2020 22:45:56 +0300 +Subject: [PATCH] bpo-39360: Ensure all workers exit when finalizing a + multiprocessing Pool (GH-19009) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +When the pull is not used via the context manager or terminate() is called, there is a system in multiprocessing.util that handles finalization of all pools via an atexit handler (the Finalize) class. This class registers the _terminate_pool handler in the registry of finalizers of the module, and that registry is called on interpreter exit via _exit_function. The problem is that the "happy" path with the context manager or manual call to finalize() does some extra steps that _terminate_pool does not. The step that is not executed when the atexit() handler calls _terminate_pool is pinging the _change_notifier queue to unblock the maintenance threads. + +This commit moves the notification to the _terminate_pool function so is called from both code paths. + +Co-authored-by: Pablo Galindo +(cherry picked from commit ac10e0c93218627d1a639db0b7b41714c5f6a883) + +Co-authored-by: Batuhan Taşkaya <47358913+isidentical@users.noreply.github.com> +--- + Lib/multiprocessing/pool.py | 7 +++++-- + Lib/test/_test_multiprocessing.py | 18 ++++++++++++++++++ + .../2020-03-15-05-41-05.bpo-39360.cmcU5p.rst | 4 ++++ + 3 files changed, 27 insertions(+), 2 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2020-03-15-05-41-05.bpo-39360.cmcU5p.rst + +diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py +index b223d6aa724bb..41dd923d4f974 100644 +--- a/Lib/multiprocessing/pool.py ++++ b/Lib/multiprocessing/pool.py +@@ -651,8 +651,6 @@ def close(self): + def terminate(self): + util.debug('terminating pool') + self._state = TERMINATE +- self._worker_handler._state = TERMINATE +- self._change_notifier.put(None) + self._terminate() + + def join(self): +@@ -682,7 +680,12 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier, + # this is guaranteed to only be called once + util.debug('finalizing pool') + ++ # Notify that the worker_handler state has been changed so the ++ # _handle_workers loop can be unblocked (and exited) in order to ++ # send the finalization sentinel all the workers. + worker_handler._state = TERMINATE ++ change_notifier.put(None) ++ + task_handler._state = TERMINATE + + util.debug('helping task handler/workers to finish') +diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py +index de912428737f9..e64a8e9b3b879 100644 +--- a/Lib/test/_test_multiprocessing.py ++++ b/Lib/test/_test_multiprocessing.py +@@ -2778,6 +2778,24 @@ def test_pool_worker_lifetime_early_close(self): + for (j, res) in enumerate(results): + self.assertEqual(res.get(), sqr(j)) + ++ def test_worker_finalization_via_atexit_handler_of_multiprocessing(self): ++ # tests cases against bpo-38744 and bpo-39360 ++ cmd = '''if 1: ++ from multiprocessing import Pool ++ problem = None ++ class A: ++ def __init__(self): ++ self.pool = Pool(processes=1) ++ def test(): ++ global problem ++ problem = A() ++ problem.pool.map(float, tuple(range(10))) ++ if __name__ == "__main__": ++ test() ++ ''' ++ rc, out, err = test.support.script_helper.assert_python_ok('-c', cmd) ++ self.assertEqual(rc, 0) ++ + # + # Test of creating a customized manager class + # +diff --git a/Misc/NEWS.d/next/Library/2020-03-15-05-41-05.bpo-39360.cmcU5p.rst b/Misc/NEWS.d/next/Library/2020-03-15-05-41-05.bpo-39360.cmcU5p.rst +new file mode 100644 +index 0000000000000..148878886e6ee +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2020-03-15-05-41-05.bpo-39360.cmcU5p.rst +@@ -0,0 +1,4 @@ ++Ensure all workers exit when finalizing a :class:`multiprocessing.Pool` implicitly via the module finalization ++handlers of multiprocessing. This fixes a deadlock situation that can be experienced when the Pool is not ++properly finalized via the context manager or a call to ``multiprocessing.Pool.terminate``. Patch by Batuhan Taskaya ++and Pablo Galindo. diff --git a/python3.spec b/python3.spec index c4f7db7..c32bb2d 100644 --- a/python3.spec +++ b/python3.spec @@ -42,7 +42,7 @@ Summary(tr.UTF-8): X arayüzlü, yüksek düzeyli, kabuk yorumlayıcı dili Summary(uk.UTF-8): Мова програмування дуже високого рівня з X-інтерфейсом Name: python3 Version: %{py_ver}.2 -Release: 1 +Release: 2 Epoch: 1 License: PSF Group: Development/Languages/Python @@ -55,7 +55,7 @@ Patch2: %{name}-multilib.patch Patch3: %{name}-no_cmdline_tests.patch Patch4: %{name}-makefile-location.patch Patch5: %{name}-config.patch - +Patch6: bug-38744.patch Patch7: %{name}-db.patch Patch8: %{name}-install_prefix.patch Patch9: %{name}-tests_with_pythonpath.patch @@ -490,7 +490,7 @@ Moduły testowe dla Pythona. %patch3 -p1 %patch4 -p1 %patch5 -p1 - +%patch6 -p1 %patch7 -p1 %patch8 -p1 %patch9 -p1