From 384e710a03a5b87ab9d2c71885989e544ab880ec Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Tue, 2 Aug 2022 00:20:05 +0200 Subject: [PATCH 01/10] Store kuttl XML as prow artifact --- .openshift-ci/ci_tests.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.openshift-ci/ci_tests.py b/.openshift-ci/ci_tests.py index 9f82eb42e9658..da4f9890b820e 100755 --- a/.openshift-ci/ci_tests.py +++ b/.openshift-ci/ci_tests.py @@ -55,6 +55,7 @@ class OperatorE2eTest(BaseTest): UPGRADE_TEST_TIMEOUT_SEC = 50 * 60 E2E_TEST_TIMEOUT_SEC = 50 * 60 SCORECARD_TEST_TIMEOUT_SEC = 20 * 60 + ARTIFACTS_TIMEOUT = 3 * 60 def __init__(self): self.test_outputs = [ @@ -72,7 +73,15 @@ def run(self): print("Executing operator upgrade test") self.run_with_graceful_kill( ["make", "-C", "operator", "test-upgrade"], - OperatorE2eTest.UPGRADE_TEST_TIMEOUT_SEC, + OperatorE2eTest.UPGRADE_TEST_TIMEOUT_SEC + ) + + print("Storing test-upgrade results in JUnit format") + self.run_with_graceful_kill( + ["scripts/ci/store-artifacts.sh", "store_test_results", + "operator/build/kuttl-test-artifacts", + "operator-test-results-upgrade"], + OperatorE2eTest.ARTIFACTS_TIMEOUT ) print("Executing operator e2e tests") @@ -81,6 +90,14 @@ def run(self): OperatorE2eTest.E2E_TEST_TIMEOUT_SEC, ) + print("Storing test-e2e-deployed results in JUnit format") + self.run_with_graceful_kill( + ["scripts/ci/store-artifacts.sh", "store_test_results", + "operator/build/kuttl-test-artifacts", + "operator-test-results-e2e-deployed"], + OperatorE2eTest.ARTIFACTS_TIMEOUT + ) + print("Executing Operator Bundle Scorecard tests") self.run_with_graceful_kill( [ From 03fd7fe0d75da36d5601c0db7c8055dd80df85f4 Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Wed, 7 Sep 2022 02:50:18 +0200 Subject: [PATCH 02/10] Move storing test results to post_tests.py --- .openshift-ci/ci_tests.py | 17 ----------------- .openshift-ci/post_tests.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.openshift-ci/ci_tests.py b/.openshift-ci/ci_tests.py index da4f9890b820e..43436984c4f7e 100755 --- a/.openshift-ci/ci_tests.py +++ b/.openshift-ci/ci_tests.py @@ -55,7 +55,6 @@ class OperatorE2eTest(BaseTest): UPGRADE_TEST_TIMEOUT_SEC = 50 * 60 E2E_TEST_TIMEOUT_SEC = 50 * 60 SCORECARD_TEST_TIMEOUT_SEC = 20 * 60 - ARTIFACTS_TIMEOUT = 3 * 60 def __init__(self): self.test_outputs = [ @@ -76,28 +75,12 @@ def run(self): OperatorE2eTest.UPGRADE_TEST_TIMEOUT_SEC ) - print("Storing test-upgrade results in JUnit format") - self.run_with_graceful_kill( - ["scripts/ci/store-artifacts.sh", "store_test_results", - "operator/build/kuttl-test-artifacts", - "operator-test-results-upgrade"], - OperatorE2eTest.ARTIFACTS_TIMEOUT - ) - print("Executing operator e2e tests") self.run_with_graceful_kill( ["make", "-C", "operator", "test-e2e-deployed"], OperatorE2eTest.E2E_TEST_TIMEOUT_SEC, ) - print("Storing test-e2e-deployed results in JUnit format") - self.run_with_graceful_kill( - ["scripts/ci/store-artifacts.sh", "store_test_results", - "operator/build/kuttl-test-artifacts", - "operator-test-results-e2e-deployed"], - OperatorE2eTest.ARTIFACTS_TIMEOUT - ) - print("Executing Operator Bundle Scorecard tests") self.run_with_graceful_kill( [ diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index 233cc03afaaf3..d67deb5199823 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -17,6 +17,7 @@ class PostTestsConstants: CHECK_TIMEOUT = 5 * 60 STORE_TIMEOUT = 5 * 60 FIXUP_TIMEOUT = 5 * 60 + ARTIFACTS_TIMEOUT = 3 * 60 # Where the QA tests store failure logs: # qa-tests-backend/src/main/groovy/common/Constants.groovy QA_TEST_DEBUG_LOGS = "/tmp/qa-tests-backend-logs" @@ -105,9 +106,11 @@ def __init__( collect_central_artifacts=True, check_stackrox_logs=False, artifact_destination_prefix=None, + add_test_output=False, ): super().__init__(artifact_destination_prefix=artifact_destination_prefix) self._check_stackrox_logs = check_stackrox_logs + self._add_test_output = add_test_output self.k8s_namespaces = ["stackrox", "stackrox-operator", "proxies", "squid"] self.openshift_namespaces = [ "openshift-dns", @@ -127,6 +130,8 @@ def run(self, test_outputs=None): self.collect_service_logs() if self._check_stackrox_logs: self.check_stackrox_logs() + if self._add_test_output: + self.add_test_output() self.store_artifacts(test_outputs) self.handle_run_failure() @@ -204,6 +209,14 @@ def check_stackrox_logs(self): timeout=PostTestsConstants.CHECK_TIMEOUT, ) + def add_test_output(self): + print("Storing operator test results in JUnit format") + self.run_with_best_effort( + ["scripts/ci/store-artifacts.sh", "store_test_results", + "operator/build/kuttl-test-artifacts"], + timeout=PostTestsConstants.ARTIFACTS_TIMEOUT, + ) + class CheckStackroxLogs(StoreArtifacts): """When only stackrox logs and checks are required""" From 52540a6da1b90421c6960ba0ac33d02ec38fc625 Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Tue, 20 Sep 2022 03:27:37 +0200 Subject: [PATCH 03/10] Add test_results field to BaseTest and use it in post_tests.py --- .openshift-ci/ci_tests.py | 9 +++++---- .openshift-ci/post_tests.py | 24 +++++++++++++----------- .openshift-ci/runners.py | 3 ++- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.openshift-ci/ci_tests.py b/.openshift-ci/ci_tests.py index 43436984c4f7e..7954404bc0f08 100755 --- a/.openshift-ci/ci_tests.py +++ b/.openshift-ci/ci_tests.py @@ -12,6 +12,7 @@ class BaseTest: def __init__(self): self.test_outputs = [] + self.test_results = {} def run_with_graceful_kill(self, args, timeout, post_start_hook=None): with subprocess.Popen(args) as cmd: @@ -57,10 +58,10 @@ class OperatorE2eTest(BaseTest): SCORECARD_TEST_TIMEOUT_SEC = 20 * 60 def __init__(self): - self.test_outputs = [ - "operator/build/kuttl-test-artifacts", - "operator/build/kuttl-test-artifacts-upgrade", - ] + self.test_results = { + "operator/build/kuttl-test-artifacts": "kuttl-test-artifacts", + "operator/build/kuttl-test-artifacts-upgrade": "kuttl-test-artifacts-upgrade", + } def run(self): print("Deploying operator") diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index d67deb5199823..d39538fb8e324 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -106,11 +106,11 @@ def __init__( collect_central_artifacts=True, check_stackrox_logs=False, artifact_destination_prefix=None, - add_test_output=False, + add_test_result=False, ): super().__init__(artifact_destination_prefix=artifact_destination_prefix) self._check_stackrox_logs = check_stackrox_logs - self._add_test_output = add_test_output + self._add_test_result = add_test_result self.k8s_namespaces = ["stackrox", "stackrox-operator", "proxies", "squid"] self.openshift_namespaces = [ "openshift-dns", @@ -121,7 +121,7 @@ def __init__( ] self.collect_central_artifacts = collect_central_artifacts - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): self.collect_collector_metrics() if self.collect_central_artifacts and self.wait_for_central_api(): self.get_central_debug_dump() @@ -130,8 +130,8 @@ def run(self, test_outputs=None): self.collect_service_logs() if self._check_stackrox_logs: self.check_stackrox_logs() - if self._add_test_output: - self.add_test_output() + if self._add_test_result: + self.add_test_result(test_results) self.store_artifacts(test_outputs) self.handle_run_failure() @@ -209,13 +209,15 @@ def check_stackrox_logs(self): timeout=PostTestsConstants.CHECK_TIMEOUT, ) - def add_test_output(self): + def add_test_result(self, test_results): print("Storing operator test results in JUnit format") - self.run_with_best_effort( - ["scripts/ci/store-artifacts.sh", "store_test_results", - "operator/build/kuttl-test-artifacts"], - timeout=PostTestsConstants.ARTIFACTS_TIMEOUT, - ) + for fromDir, toDir in test_results: + self.run_with_best_effort( + ["scripts/ci/store-artifacts.sh", "store_test_results", + fromDir, toDir], + timeout=PostTestsConstants.ARTIFACTS_TIMEOUT, + ) + class CheckStackroxLogs(StoreArtifacts): diff --git a/.openshift-ci/runners.py b/.openshift-ci/runners.py index 65153d0041f8c..42707fb61f230 100644 --- a/.openshift-ci/runners.py +++ b/.openshift-ci/runners.py @@ -101,7 +101,8 @@ def run_test_set(self, test_set): hold = err try: self.log_event("About to run post test", test_set) - test_set["post_test"].run(test_outputs=test_set["test"].test_outputs) + test_set["post_test"].run(test_outputs=test_set["test"].test_outputs, + test_results=test_set["test"].test_results) self.log_event("post test completed", test_set) except Exception as err: self.log_event("ERROR: post test failed", test_set) From 8fe56627f3ec8ceb6b2627485dd25502f89a67f7 Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Tue, 20 Sep 2022 03:34:02 +0200 Subject: [PATCH 04/10] Set add_test_result to True by default --- .openshift-ci/post_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index d39538fb8e324..e1ec2fcede051 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -106,7 +106,7 @@ def __init__( collect_central_artifacts=True, check_stackrox_logs=False, artifact_destination_prefix=None, - add_test_result=False, + add_test_result=True, ): super().__init__(artifact_destination_prefix=artifact_destination_prefix) self._check_stackrox_logs = check_stackrox_logs From f9436ada13c55a186f1fb75a52fc293799168d91 Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Thu, 22 Sep 2022 14:37:51 +0200 Subject: [PATCH 05/10] Add test_results parameter to all post test --- .openshift-ci/post_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index e1ec2fcede051..0987da5f71a75 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -31,7 +31,7 @@ class PostTestsConstants: class NullPostTest: - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): pass @@ -76,7 +76,7 @@ def __init__( self.artifact_destination_prefix = artifact_destination_prefix self.data_to_store = [] - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): self.store_artifacts(test_outputs) self.handle_run_failure() @@ -234,7 +234,7 @@ def __init__( self._check_for_errors_in_stackrox_logs = check_for_errors_in_stackrox_logs self.central_is_responsive = False - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): self.central_is_responsive = self.wait_for_central_api() if self.central_is_responsive: self.collect_stackrox_logs() @@ -302,7 +302,7 @@ def __init__( self.data_to_store.append(PostTestsConstants.QA_SPOCK_RESULTS) self._handle_e2e_progress_failures = handle_e2e_progress_failures - def run(self, test_outputs=None): + def run(self, test_outputs=None, test_results=None): self.store_artifacts() self.fixup_artifacts_content_type() self.make_artifacts_help() From 05dcb5b4fa4c0b67907690631fd1f4f756636066 Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Wed, 5 Oct 2022 13:30:01 +0200 Subject: [PATCH 06/10] Address Marcin's feedback --- .openshift-ci/ci_tests.py | 8 +++-- .openshift-ci/post_tests.py | 61 ++++++++++++++++++------------------- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/.openshift-ci/ci_tests.py b/.openshift-ci/ci_tests.py index 7954404bc0f08..1d3156cc04c8a 100755 --- a/.openshift-ci/ci_tests.py +++ b/.openshift-ci/ci_tests.py @@ -58,9 +58,13 @@ class OperatorE2eTest(BaseTest): SCORECARD_TEST_TIMEOUT_SEC = 20 * 60 def __init__(self): + self.test_outputs = [ + "operator/build/kuttl-test-artifacts", + "operator/build/kuttl-test-artifacts-upgrade", + ] self.test_results = { - "operator/build/kuttl-test-artifacts": "kuttl-test-artifacts", - "operator/build/kuttl-test-artifacts-upgrade": "kuttl-test-artifacts-upgrade", + "kuttl-test-artifacts": "operator/build/kuttl-test-artifacts", + "kuttl-test-artifacts-upgrade": "operator/build/kuttl-test-artifacts-upgrade", } def run(self): diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index 0987da5f71a75..a3612ba68b19f 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -11,7 +11,6 @@ class PostTestsConstants: - API_TIMEOUT = 5 * 60 COLLECT_TIMEOUT = 10 * 60 CHECK_TIMEOUT = 5 * 60 @@ -37,7 +36,7 @@ def run(self, test_outputs=None, test_results=None): class RunWithBestEffortMixin: def __init__( - self, + self, ): self.exitstatus = 0 self.failed_commands: List[List[str]] = [] @@ -69,8 +68,8 @@ class StoreArtifacts(RunWithBestEffortMixin): """For tests that only need to store artifacts""" def __init__( - self, - artifact_destination_prefix=None, + self, + artifact_destination_prefix=None, ): super().__init__() self.artifact_destination_prefix = artifact_destination_prefix @@ -78,8 +77,20 @@ def __init__( def run(self, test_outputs=None, test_results=None): self.store_artifacts(test_outputs) + self.add_test_results(test_results) self.handle_run_failure() + def add_test_results(self, test_results): + if test_results is None: + return + print("Storing test results in JUnit format") + for to_dir, from_dir in test_results: + self.run_with_best_effort( + ["scripts/ci/store-artifacts.sh", "store_test_results", + from_dir, to_dir], + timeout=PostTestsConstants.ARTIFACTS_TIMEOUT, + ) + def store_artifacts(self, test_outputs=None): if test_outputs is not None: self.data_to_store = test_outputs + self.data_to_store @@ -102,15 +113,13 @@ class PostClusterTest(StoreArtifacts): """The standard cluster test suite of debug gathering and analysis""" def __init__( - self, - collect_central_artifacts=True, - check_stackrox_logs=False, - artifact_destination_prefix=None, - add_test_result=True, + self, + collect_central_artifacts=True, + check_stackrox_logs=False, + artifact_destination_prefix=None, ): super().__init__(artifact_destination_prefix=artifact_destination_prefix) self._check_stackrox_logs = check_stackrox_logs - self._add_test_result = add_test_result self.k8s_namespaces = ["stackrox", "stackrox-operator", "proxies", "squid"] self.openshift_namespaces = [ "openshift-dns", @@ -130,8 +139,7 @@ def run(self, test_outputs=None, test_results=None): self.collect_service_logs() if self._check_stackrox_logs: self.check_stackrox_logs() - if self._add_test_result: - self.add_test_result(test_results) + self.add_test_results(test_results) self.store_artifacts(test_outputs) self.handle_run_failure() @@ -209,25 +217,15 @@ def check_stackrox_logs(self): timeout=PostTestsConstants.CHECK_TIMEOUT, ) - def add_test_result(self, test_results): - print("Storing operator test results in JUnit format") - for fromDir, toDir in test_results: - self.run_with_best_effort( - ["scripts/ci/store-artifacts.sh", "store_test_results", - fromDir, toDir], - timeout=PostTestsConstants.ARTIFACTS_TIMEOUT, - ) - - class CheckStackroxLogs(StoreArtifacts): """When only stackrox logs and checks are required""" def __init__( - self, - check_for_stackrox_restarts=False, - check_for_errors_in_stackrox_logs=False, - artifact_destination_prefix=None, + self, + check_for_stackrox_restarts=False, + check_for_errors_in_stackrox_logs=False, + artifact_destination_prefix=None, ): super().__init__(artifact_destination_prefix=artifact_destination_prefix) self._check_for_stackrox_restarts = check_for_stackrox_restarts @@ -243,6 +241,7 @@ def run(self, test_outputs=None, test_results=None): if self._check_for_errors_in_stackrox_logs: self.check_for_errors_in_stackrox_logs() self.store_artifacts(test_outputs) + self.add_test_results(test_results) self.handle_run_failure() def wait_for_central_api(self): @@ -287,11 +286,11 @@ class FinalPost(StoreArtifacts): """Collect logs that accumulate over multiple tests and other final steps""" def __init__( - self, - store_qa_test_debug_logs=False, - store_qa_spock_results=False, - artifact_destination_prefix="final", - handle_e2e_progress_failures=True, + self, + store_qa_test_debug_logs=False, + store_qa_spock_results=False, + artifact_destination_prefix="final", + handle_e2e_progress_failures=True, ): super().__init__(artifact_destination_prefix=artifact_destination_prefix) self._store_qa_test_debug_logs = store_qa_test_debug_logs From ad4e3100325e37c394f70e128323f8dd421da445 Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Wed, 5 Oct 2022 13:37:07 +0200 Subject: [PATCH 07/10] Fix FinalPost --- .openshift-ci/post_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index a3612ba68b19f..d405b9243dc89 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -303,6 +303,7 @@ def __init__( def run(self, test_outputs=None, test_results=None): self.store_artifacts() + self.add_test_results(test_results) self.fixup_artifacts_content_type() self.make_artifacts_help() self.handle_run_failure() From 014ba59c98bd7aca0ff2885f899f3a802edbceed Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Thu, 6 Oct 2022 23:24:36 +0200 Subject: [PATCH 08/10] Fix for-loop --- .openshift-ci/post_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index d405b9243dc89..1406e7ac152e1 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -84,7 +84,7 @@ def add_test_results(self, test_results): if test_results is None: return print("Storing test results in JUnit format") - for to_dir, from_dir in test_results: + for to_dir, from_dir in test_results.items(): self.run_with_best_effort( ["scripts/ci/store-artifacts.sh", "store_test_results", from_dir, to_dir], From 324572c9d4df238ba4f0ac85e08d2a17c9055529 Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Fri, 7 Oct 2022 14:16:20 +0200 Subject: [PATCH 09/10] Address Marcin's final comments. --- .openshift-ci/post_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py index 1406e7ac152e1..bf6cc83deb15e 100644 --- a/.openshift-ci/post_tests.py +++ b/.openshift-ci/post_tests.py @@ -81,7 +81,7 @@ def run(self, test_outputs=None, test_results=None): self.handle_run_failure() def add_test_results(self, test_results): - if test_results is None: + if not test_results: return print("Storing test results in JUnit format") for to_dir, from_dir in test_results.items(): @@ -139,8 +139,8 @@ def run(self, test_outputs=None, test_results=None): self.collect_service_logs() if self._check_stackrox_logs: self.check_stackrox_logs() - self.add_test_results(test_results) self.store_artifacts(test_outputs) + self.add_test_results(test_results) self.handle_run_failure() def wait_for_central_api(self): From 2cd25e97a18bc5c6489c99772bdf0e2f55b183f6 Mon Sep 17 00:00:00 2001 From: Marcin Owsiany Date: Mon, 10 Oct 2022 07:06:32 +0200 Subject: [PATCH 10/10] Provoke a failure --- operator/tests/securedcluster/basic-sc/30-assert.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/tests/securedcluster/basic-sc/30-assert.yaml b/operator/tests/securedcluster/basic-sc/30-assert.yaml index ae9f168c446a1..ddc10504432b2 100644 --- a/operator/tests/securedcluster/basic-sc/30-assert.yaml +++ b/operator/tests/securedcluster/basic-sc/30-assert.yaml @@ -3,7 +3,7 @@ kind: SecuredCluster metadata: name: stackrox-secured-cluster-services status: - clusterName: testing-cluster + clusterName: testing-cluster-this-should-fail conditions: - status: "True" type: Deployed