diff --git a/.openshift-ci/ci_tests.py b/.openshift-ci/ci_tests.py index 9f82eb42e9658..1d3156cc04c8a 100755 --- a/.openshift-ci/ci_tests.py +++ b/.openshift-ci/ci_tests.py @@ -12,6 +12,7 @@ class BaseTest: def __init__(self): self.test_outputs = [] + self.test_results = {} def run_with_graceful_kill(self, args, timeout, post_start_hook=None): with subprocess.Popen(args) as cmd: @@ -61,6 +62,10 @@ def __init__(self): "operator/build/kuttl-test-artifacts", "operator/build/kuttl-test-artifacts-upgrade", ] + self.test_results = { + "kuttl-test-artifacts": "operator/build/kuttl-test-artifacts", + "kuttl-test-artifacts-upgrade": "operator/build/kuttl-test-artifacts-upgrade", + } def run(self): print("Deploying operator") @@ -72,7 +77,7 @@ def run(self): print("Executing operator upgrade test") self.run_with_graceful_kill( ["make", "-C", "operator", "test-upgrade"], - OperatorE2eTest.UPGRADE_TEST_TIMEOUT_SEC, + OperatorE2eTest.UPGRADE_TEST_TIMEOUT_SEC ) print("Executing operator e2e tests") diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index 233cc03afaaf3..bf6cc83deb15e 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -11,12 +11,12 @@ class PostTestsConstants: - API_TIMEOUT = 5 * 60 COLLECT_TIMEOUT = 10 * 60 CHECK_TIMEOUT = 5 * 60 STORE_TIMEOUT = 5 * 60 FIXUP_TIMEOUT = 5 * 60 + ARTIFACTS_TIMEOUT = 3 * 60 # Where the QA tests store failure logs: # qa-tests-backend/src/main/groovy/common/Constants.groovy QA_TEST_DEBUG_LOGS = "/tmp/qa-tests-backend-logs" @@ -30,13 +30,13 @@ class PostTestsConstants: class NullPostTest: - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): pass class RunWithBestEffortMixin: def __init__( - self, + self, ): self.exitstatus = 0 self.failed_commands: List[List[str]] = [] @@ -68,17 +68,29 @@ class StoreArtifacts(RunWithBestEffortMixin): """For tests that only need to store artifacts""" def __init__( - self, - artifact_destination_prefix=None, + self, + artifact_destination_prefix=None, ): super().__init__() self.artifact_destination_prefix = artifact_destination_prefix self.data_to_store = [] - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): self.store_artifacts(test_outputs) + self.add_test_results(test_results) self.handle_run_failure() + def add_test_results(self, test_results): + if not test_results: + return + print("Storing test results in JUnit format") + for to_dir, from_dir in test_results.items(): + self.run_with_best_effort( + ["scripts/ci/store-artifacts.sh", "store_test_results", + from_dir, to_dir], + timeout=PostTestsConstants.ARTIFACTS_TIMEOUT, + ) + def store_artifacts(self, test_outputs=None): if test_outputs is not None: self.data_to_store = test_outputs + self.data_to_store @@ -101,10 +113,10 @@ class PostClusterTest(StoreArtifacts): """The standard cluster test suite of debug gathering and analysis""" def __init__( - self, - collect_central_artifacts=True, - check_stackrox_logs=False, - artifact_destination_prefix=None, + self, + collect_central_artifacts=True, + check_stackrox_logs=False, + artifact_destination_prefix=None, ): super().__init__(artifact_destination_prefix=artifact_destination_prefix) self._check_stackrox_logs = check_stackrox_logs @@ -118,7 +130,7 @@ def __init__( ] self.collect_central_artifacts = collect_central_artifacts - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): self.collect_collector_metrics() if self.collect_central_artifacts and self.wait_for_central_api(): self.get_central_debug_dump() @@ -128,6 +140,7 @@ def run(self, test_outputs=None): if self._check_stackrox_logs: self.check_stackrox_logs() self.store_artifacts(test_outputs) + self.add_test_results(test_results) self.handle_run_failure() def wait_for_central_api(self): @@ -209,17 +222,17 @@ class CheckStackroxLogs(StoreArtifacts): """When only stackrox logs and checks are required""" def __init__( - self, - check_for_stackrox_restarts=False, - check_for_errors_in_stackrox_logs=False, - artifact_destination_prefix=None, + self, + check_for_stackrox_restarts=False, + check_for_errors_in_stackrox_logs=False, + artifact_destination_prefix=None, ): super().__init__(artifact_destination_prefix=artifact_destination_prefix) self._check_for_stackrox_restarts = check_for_stackrox_restarts self._check_for_errors_in_stackrox_logs = check_for_errors_in_stackrox_logs self.central_is_responsive = False - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): self.central_is_responsive = self.wait_for_central_api() if self.central_is_responsive: self.collect_stackrox_logs() @@ -228,6 +241,7 @@ def run(self, test_outputs=None): if self._check_for_errors_in_stackrox_logs: self.check_for_errors_in_stackrox_logs() self.store_artifacts(test_outputs) + self.add_test_results(test_results) self.handle_run_failure() def wait_for_central_api(self): @@ -272,11 +286,11 @@ class FinalPost(StoreArtifacts): """Collect logs that accumulate over multiple tests and other final steps""" def __init__( - self, - store_qa_test_debug_logs=False, - store_qa_spock_results=False, - artifact_destination_prefix="final", - handle_e2e_progress_failures=True, + self, + store_qa_test_debug_logs=False, + store_qa_spock_results=False, + artifact_destination_prefix="final", + handle_e2e_progress_failures=True, ): super().__init__(artifact_destination_prefix=artifact_destination_prefix) self._store_qa_test_debug_logs = store_qa_test_debug_logs @@ -287,8 +301,9 @@ def __init__( self.data_to_store.append(PostTestsConstants.QA_SPOCK_RESULTS) self._handle_e2e_progress_failures = handle_e2e_progress_failures - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): self.store_artifacts() + self.add_test_results(test_results) self.fixup_artifacts_content_type() self.make_artifacts_help() self.handle_run_failure() diff --git a/.openshift-ci/runners.py b/.openshift-ci/runners.py index 65153d0041f8c..42707fb61f230 100644 --- a/.openshift-ci/runners.py +++ b/.openshift-ci/runners.py @@ -101,7 +101,8 @@ def run_test_set(self, test_set): hold = err try: self.log_event("About to run post test", test_set) - test_set["post_test"].run(test_outputs=test_set["test"].test_outputs) + test_set["post_test"].run(test_outputs=test_set["test"].test_outputs, + test_results=test_set["test"].test_results) self.log_event("post test completed", test_set) except Exception as err: self.log_event("ERROR: post test failed", test_set) diff --git a/operator/tests/securedcluster/basic-sc/30-assert.yaml b/operator/tests/securedcluster/basic-sc/30-assert.yaml index ae9f168c446a1..ddc10504432b2 100644 --- a/operator/tests/securedcluster/basic-sc/30-assert.yaml +++ b/operator/tests/securedcluster/basic-sc/30-assert.yaml @@ -3,7 +3,7 @@ kind: SecuredCluster metadata: name: stackrox-secured-cluster-services status: - clusterName: testing-cluster + clusterName: testing-cluster-this-should-fail conditions: - status: "True" type: Deployed