From be36f520c1898c24774f9d6f63ef82afc69649de Mon Sep 17 00:00:00 2001 From: Maksim Stepanov <17935127+delatrie@users.noreply.github.com> Date: Wed, 30 Nov 2022 01:23:23 +0700 Subject: [PATCH 1/4] Add test to catch multithreaded concurrency error --- allure-pytest/pyproject.toml | 2 +- allure-pytest/requirements.txt | 1 + .../acceptance/capture/capture_attach_test.py | 7 +- .../test/acceptance/concurrency/__init__.py | 0 .../multithreaded_concurrency_test.py | 106 ++++++++++++++++++ 5 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 allure-pytest/test/acceptance/concurrency/__init__.py create mode 100644 allure-pytest/test/acceptance/concurrency/multithreaded_concurrency_test.py diff --git a/allure-pytest/pyproject.toml b/allure-pytest/pyproject.toml index b7ba99a6..ed010ce4 100644 --- a/allure-pytest/pyproject.toml +++ b/allure-pytest/pyproject.toml @@ -1,3 +1,3 @@ [tool.poe.tasks] linter = "flake8 ./src ./test" -tests = "pytest --alluredir=allure-results --basetemp=tmp test/acceptance test/integration" +tests = "pytest --alluredir=allure-results --basetemp=tmp test/acceptance test/integration -p no:asyncio-cooperative" diff --git a/allure-pytest/requirements.txt b/allure-pytest/requirements.txt index f5585f82..fe06dbed 100644 --- a/allure-pytest/requirements.txt +++ b/allure-pytest/requirements.txt @@ -6,6 +6,7 @@ pytest-flakes pytest-rerunfailures pytest-xdist pytest-lazy-fixture +pytest-asyncio-cooperative poethepoet # linters flake8==5.* diff --git a/allure-pytest/test/acceptance/capture/capture_attach_test.py b/allure-pytest/test/acceptance/capture/capture_attach_test.py index b438eb79..ef1029d2 100644 --- a/allure-pytest/test/acceptance/capture/capture_attach_test.py +++ b/allure-pytest/test/acceptance/capture/capture_attach_test.py @@ -93,7 +93,12 @@ def test_capture_log(allured_testdir, logging): params = [] if logging else ["-p", "no:logging"] if_logging_ = is_ if logging else is_not - allured_testdir.run_with_allure("--log-cli-level=INFO", *params) + allured_testdir.run_with_allure( + "--log-cli-level=INFO", + "-p", + "no:asyncio-cooperative", + *params + ) assert_that(allured_testdir.allure_report, has_property("attachments", diff --git a/allure-pytest/test/acceptance/concurrency/__init__.py b/allure-pytest/test/acceptance/concurrency/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/allure-pytest/test/acceptance/concurrency/multithreaded_concurrency_test.py b/allure-pytest/test/acceptance/concurrency/multithreaded_concurrency_test.py new file mode 100644 index 00000000..eff8be34 --- /dev/null +++ b/allure-pytest/test/acceptance/concurrency/multithreaded_concurrency_test.py @@ -0,0 +1,106 @@ +from concurrent.futures import ThreadPoolExecutor +from hamcrest import assert_that, has_entry, has_length, has_property, all_of +from allure_commons.logger import AllureMemoryLogger +import allure_pytest +from ...conftest import fake_logger +import allure_commons +from allure_commons_test.report import has_test_case +from allure_commons_test.result import has_step +from allure_commons_test.result import with_status + + +def test_concurrent_mt_pytest_runs(allured_testdir, monkeypatch): + """ + >>> import allure + >>> import pytest + + >>> @pytest.mark.thread1 + ... def test_multithreaded_concurrency_fn1(): + ... with allure.step("Step 1 of test 1"): + ... pass + ... with allure.step("Step 2 of test 1"): + ... pass + + >>> @pytest.mark.thread2 + ... def test_multithreaded_concurrency_fn2(): + ... with allure.step("Step 1 of test 2"): + ... pass + ... with allure.step("Step 2 of test 2"): + ... pass + """ + + thread_pool = ThreadPoolExecutor(max_workers=2) + + allured_testdir.parse_docstring_source() + allured_testdir.allure_report = AllureMemoryLogger() + + original_register = allure_commons.plugin_manager.register + + def register_nothrow(*args, **kwargs): + try: + return original_register(*args, **kwargs) + except Exception: + pass + + monkeypatch.setattr(allure_commons.plugin_manager, "register", register_nothrow) + + original_cleanup_factory = allure_pytest.plugin.cleanup_factory + + def cleanup_factory_nothrow(*args, **kwargs): + cleanup = original_cleanup_factory(*args, **kwargs) + + def wrapped_cleanup(): + try: + cleanup() + except Exception: + pass + return wrapped_cleanup + + monkeypatch.setattr(allure_pytest.plugin, "cleanup_factory", cleanup_factory_nothrow) + + with fake_logger("allure_pytest.plugin.AllureFileLogger", allured_testdir.allure_report): + for n in [1, 2]: + thread_pool.submit( + allured_testdir.testdir.runpytest, + "--alluredir", + allured_testdir.testdir.tmpdir, + "-m", + f"thread{n}", + "--disable-warnings", + "-q" + ) + thread_pool.shutdown(True) + + assert_that( + allured_testdir.allure_report, + all_of( + has_property("test_cases", has_length(2)), + has_test_case( + "test_multithreaded_concurrency_fn1", + with_status("passed"), + has_entry("steps", has_length(2)), + has_step( + "Step 1 of test 1", + with_status("passed") + ), + has_step( + "Step 2 of test 1", + with_status("passed") + ) + ), + has_test_case( + "test_multithreaded_concurrency_fn2", + with_status("passed"), + has_entry("steps", has_length(2)), + has_step( + "Step 1 of test 2", + with_status("passed") + ), + has_step( + "Step 2 of test 2", + with_status("passed") + ) + ) + ), + "Should contain two passed tests cases with two steps each" + ) From 62387a6ff394c64e35b340fa4ed819d2be0af283 Mon Sep 17 00:00:00 2001 From: Maksim Stepanov <17935127+delatrie@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:35:10 +0700 Subject: [PATCH 2/4] Add test to catch error with async concurrent steps --- .../async_concurrent_steps_test.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 allure-pytest/test/acceptance/concurrency/async_concurrent_steps_test.py diff --git a/allure-pytest/test/acceptance/concurrency/async_concurrent_steps_test.py b/allure-pytest/test/acceptance/concurrency/async_concurrent_steps_test.py new file mode 100644 index 00000000..62b062ef --- /dev/null +++ b/allure-pytest/test/acceptance/concurrency/async_concurrent_steps_test.py @@ -0,0 +1,49 @@ +from hamcrest import assert_that, has_entry, has_length +from allure_commons_test.report import has_only_n_test_cases +from allure_commons_test.result import has_step +from allure_commons_test.result import with_status + +def test_steps_could_be_run_cuncurrently_with_asyncio(executed_docstring_source): + """ + >>> import allure + >>> import pytest + >>> import asyncio + + >>> async def run_steps(name, to_set, to_wait): + ... with allure.step(f"{name}-step"): + ... to_set.set() + ... await to_wait.wait() + + >>> async def run_tasks(): + ... task1_fence = asyncio.Event() + ... task2_fence = asyncio.Event() + ... await asyncio.gather( + ... run_steps("task1", task2_fence, task1_fence), + ... run_steps("task2", task1_fence, task2_fence) + ... ) + + >>> def test_asyncio_concurrency(): + ... asyncio.get_event_loop().run_until_complete( + ... run_tasks() + ... ) + """ + + assert_that( + executed_docstring_source.allure_report, + has_only_n_test_cases( + "test_asyncio_concurrency", + 1, + with_status("passed"), + has_entry("steps", has_length(2)), + has_step( + "task1-step", + with_status("passed") + ), + has_step( + "task2-step", + with_status("passed") + ) + ), + "Should contain a single test with two non-nested steps" + ) + From e18d83aa90f2907c237e73f96b40b55323ade1f3 Mon Sep 17 00:00:00 2001 From: Maksim Stepanov <17935127+delatrie@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:38:26 +0700 Subject: [PATCH 3/4] Add a unittest to catch multithreaded tests error --- .../test/mt_concurrency_test.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 allure-python-commons/test/mt_concurrency_test.py diff --git a/allure-python-commons/test/mt_concurrency_test.py b/allure-python-commons/test/mt_concurrency_test.py new file mode 100644 index 00000000..60ba37ca --- /dev/null +++ b/allure-python-commons/test/mt_concurrency_test.py @@ -0,0 +1,90 @@ +import pytest +import allure_commons +from concurrent.futures import ThreadPoolExecutor +from allure_commons.reporter import AllureReporter +from allure_commons.utils import uuid4 +from allure_commons.model2 import TestResult, TestStepResult +from allure_commons.logger import AllureMemoryLogger + + +@pytest.fixture +def allure_results(): + logger = AllureMemoryLogger() + plugin_id = allure_commons.plugin_manager.register(logger) + yield logger + allure_commons.plugin_manager.unregister(plugin_id) + +def __generate_test(name, start): + return { + "uuid": uuid4(), + "name": name, + "fullName": f"module#{name}", + "status": "passed", + "start": start, + "stop": start + 100, + "historyId": uuid4(), + "testCaseId": uuid4(), + "steps": [ + { + "name": f"step-1 of {name}", + "status": "passed", + "start": start + 10, + "stop": start + 40 + }, + { + "name": f"step-1 of {name}", + "status": "passed", + "start": start + 50, + "stop": start + 90 + } + ] + } + +def test_state_not_corrupted_in_mt_env(allure_results): + reporter = AllureReporter() + + def run_tests(thread, test_count): + for index, start in enumerate(range(0, 100 * test_count, 100), 1): + test = __generate_test(f"thread_{thread}_test_{index}", start) + reporter.schedule_test( + test["uuid"], + TestResult( + name=test["name"], + status=test["status"], + start=test["start"], + stop=test["stop"], + uuid=test["uuid"], + fullName=test["fullName"], + historyId=test["historyId"], + testCaseId=test["testCaseId"] + ) + ) + for step in test["steps"]: + step_uuid = uuid4() + reporter.start_step( + None, + step_uuid, + TestStepResult( + name=step["name"], + start=step["start"] + ) + ) + reporter.stop_step( + step_uuid, + stop=step["stop"], + status=step["status"] + ) + reporter.close_test(test["uuid"]) + + futures = [] + with ThreadPoolExecutor(max_workers=2) as pool: + futures.extend([ + pool.submit(run_tests, 1, 2), + pool.submit(run_tests, 2, 2) + ]) + pool.shutdown(True) + + for future in futures: + assert future.done() + assert future.result() is None + assert len(allure_results.test_cases) == 4 From a86a8f4172681257bc64eb3ff8feeb78592090c2 Mon Sep 17 00:00:00 2001 From: Maksim Stepanov <17935127+delatrie@users.noreply.github.com> Date: Thu, 1 Dec 2022 15:15:17 +0700 Subject: [PATCH 4/4] Add a test to catch async concurrency error --- .../concurrency/async_concurrency_test.py | 99 +++++++++++++++++++ .../async_concurrent_steps_test.py | 49 --------- 2 files changed, 99 insertions(+), 49 deletions(-) create mode 100644 allure-pytest/test/acceptance/concurrency/async_concurrency_test.py delete mode 100644 allure-pytest/test/acceptance/concurrency/async_concurrent_steps_test.py diff --git a/allure-pytest/test/acceptance/concurrency/async_concurrency_test.py b/allure-pytest/test/acceptance/concurrency/async_concurrency_test.py new file mode 100644 index 00000000..f622e796 --- /dev/null +++ b/allure-pytest/test/acceptance/concurrency/async_concurrency_test.py @@ -0,0 +1,99 @@ +from hamcrest import assert_that, has_entry, has_length, all_of, has_property +from allure_commons_test.report import has_test_case +from allure_commons_test.report import has_only_n_test_cases +from allure_commons_test.result import has_step +from allure_commons_test.result import with_status + +def test_asyncio_concurrent_steps(executed_docstring_source): + """ + >>> import allure + >>> import asyncio + + >>> async def run_steps(name, to_set, to_wait): + ... with allure.step(f"{name}-step"): + ... to_set.set() + ... await to_wait.wait() + + >>> async def run_tasks(): + ... task1_fence = asyncio.Event() + ... task2_fence = asyncio.Event() + ... await asyncio.gather( + ... run_steps("task1", task2_fence, task1_fence), + ... run_steps("task2", task1_fence, task2_fence) + ... ) + + >>> def test_asyncio_concurrency(): + ... asyncio.get_event_loop().run_until_complete( + ... run_tasks() + ... ) + """ + + assert_that( + executed_docstring_source.allure_report, + has_only_n_test_cases( + "test_asyncio_concurrency", + 1, + with_status("passed"), + has_entry("steps", has_length(2)), + has_step( + "task1-step", + with_status("passed") + ), + has_step( + "task2-step", + with_status("passed") + ) + ), + "Should contain a single test with two non-nested steps" + ) + + +def test_async_concurrent_tests(allured_testdir): + """ + >>> import allure + >>> import pytest + ... import asyncio + + >>> @pytest.fixture(scope="module") + ... async def event(): + ... return asyncio.Event() + + >>> @pytest.mark.asyncio_cooperative + >>> async def test_async_concurrency_fn1(event): + ... with allure.step("Step of fn1"): + ... await event.wait() + + >>> @pytest.mark.asyncio_cooperative + >>> async def test_async_concurrency_fn2(event): + ... with allure.step("Step of fn2"): + ... event.set() + """ + + allured_testdir.parse_docstring_source() + allured_testdir.run_with_allure("-q", "-p", "asyncio-cooperative") + + assert_that( + allured_testdir.allure_report, + all_of( + has_property("test_cases", has_length(2)), + has_test_case( + "test_async_concurrency_fn1", + with_status("passed"), + has_entry("steps", has_length(1)), + has_step( + "Step of fn1", + with_status("passed") + ) + ), + has_test_case( + "test_async_concurrency_fn2", + with_status("passed"), + has_entry("steps", has_length(1)), + has_step( + "Step of fn2", + with_status("passed") + ) + ) + ), + "Should contain two passed tests cases with two steps each" + ) \ No newline at end of file diff --git a/allure-pytest/test/acceptance/concurrency/async_concurrent_steps_test.py b/allure-pytest/test/acceptance/concurrency/async_concurrent_steps_test.py deleted file mode 100644 index 62b062ef..00000000 --- a/allure-pytest/test/acceptance/concurrency/async_concurrent_steps_test.py +++ /dev/null @@ -1,49 +0,0 @@ -from hamcrest import assert_that, has_entry, has_length -from allure_commons_test.report import has_only_n_test_cases -from allure_commons_test.result import has_step -from allure_commons_test.result import with_status - -def test_steps_could_be_run_cuncurrently_with_asyncio(executed_docstring_source): - """ - >>> import allure - >>> import pytest - >>> import asyncio - - >>> async def run_steps(name, to_set, to_wait): - ... with allure.step(f"{name}-step"): - ... to_set.set() - ... await to_wait.wait() - - >>> async def run_tasks(): - ... task1_fence = asyncio.Event() - ... task2_fence = asyncio.Event() - ... await asyncio.gather( - ... run_steps("task1", task2_fence, task1_fence), - ... run_steps("task2", task1_fence, task2_fence) - ... ) - - >>> def test_asyncio_concurrency(): - ... asyncio.get_event_loop().run_until_complete( - ... run_tasks() - ... ) - """ - - assert_that( - executed_docstring_source.allure_report, - has_only_n_test_cases( - "test_asyncio_concurrency", - 1, - with_status("passed"), - has_entry("steps", has_length(2)), - has_step( - "task1-step", - with_status("passed") - ), - has_step( - "task2-step", - with_status("passed") - ) - ), - "Should contain a single test with two non-nested steps" - ) -