From 58b5265b636862fbbfb916055a598c2e4ad5a2da Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Tue, 24 Mar 2020 11:07:22 -0400 Subject: [PATCH 1/8] Release 0.11.0 --- openshift/__init__.py | 4 ++-- python-openshift.spec | 2 +- requirements.txt | 4 ++-- scripts/constants.py | 4 ++-- setup.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/openshift/__init__.py b/openshift/__init__.py index c40d6957..a7eb8f9f 100644 --- a/openshift/__init__.py +++ b/openshift/__init__.py @@ -14,5 +14,5 @@ # Do not edit these constants. They will be updated automatically # by scripts/update-version.sh. -__version__ = "0.10.0dev1" -__k8s_client_version__ = "9.0.0" +__version__ = "0.11.0" +__k8s_client_version__ = "11.0.0" diff --git a/python-openshift.spec b/python-openshift.spec index b3e83a9b..32980f4a 100644 --- a/python-openshift.spec +++ b/python-openshift.spec @@ -16,7 +16,7 @@ %endif Name: python-%{library} -Version: 0.10.0dev1 +Version: 0.11.0 Release: 1%{?dist} Summary: Python client for the OpenShift API License: ASL 2.0 diff --git a/requirements.txt b/requirements.txt index a4bf15be..7147b662 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ dictdiffer jinja2 -kubernetes +kubernetes ~= 11.0 python-string-utils -ruamel.yaml +ruamel.yaml >= 0.15 six diff --git a/scripts/constants.py b/scripts/constants.py index 8c22e5e8..43dddb41 100644 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -23,8 +23,8 @@ # client version for packaging and releasing. It can # be different than SPEC_VERSION. -CLIENT_VERSION = "0.10.0dev1" -KUBERNETES_CLIENT_VERSION = "10.0.1" +CLIENT_VERSION = "0.11.0" +KUBERNETES_CLIENT_VERSION = "11.0.0" # Name of the release package PACKAGE_NAME = "openshift" diff --git a/setup.py b/setup.py index d2b794c1..7ca5d897 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ # Do not edit these constants. They will be updated automatically # by scripts/update-client.sh. -CLIENT_VERSION = "0.10.0dev1" +CLIENT_VERSION = "0.11.0" PACKAGE_NAME = "openshift" DEVELOPMENT_STATUS = "3 - Alpha" From d2576725b48a3db864652e6e57b6dfc794d42e32 Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Tue, 24 Mar 2020 12:22:52 -0400 Subject: [PATCH 2/8] Fix travis condition --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b42e83d7..1d67dbdf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ jobs: on: tags: true repo: openshift/openshift-restclient-python - condition: "$TRAVIS_TAG =~ ^v[0-9]\\.[0-9]\\.[0-9](([ab]|dev|rc)[0-9])?$" + condition: "$TRAVIS_TAG =~ ^v[0-9]+\\.[0-9]+\\.[0-9]+(([ab]|dev|rc)[0-9]+)?$" - stage: test-deploy script: python -c "import openshift ; print(openshift.__version__)" install: From c0059f4aff2a38dc0434e98f3f6e960d7fc8013c Mon Sep 17 00:00:00 2001 From: OpenShift Cherrypick Robot Date: Fri, 15 May 2020 18:55:26 +0200 Subject: [PATCH 3/8] Improve handling of dict patches in lists (#364) List items should be properly recursively patched, otherwise changes to nested fields can be ignored. This does change one of the tests to allow the deeper patching to work correctly Co-authored-by: Will Thames --- openshift/dynamic/apply.py | 9 ++++----- test/unit/test_apply.py | 24 +++++++++++++++++++++++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/openshift/dynamic/apply.py b/openshift/dynamic/apply.py index b975b950..3b0ef325 100644 --- a/openshift/dynamic/apply.py +++ b/openshift/dynamic/apply.py @@ -129,9 +129,9 @@ def apply(resource, definition): # from last_applied to desired. To find it, we compute deletions, which are the deletions from # last_applied to desired, and delta, which is the difference from actual to desired without # deletions, and then apply delta to deletions as a patch, which should be strictly additive. -def merge(last_applied, desired, actual): +def merge(last_applied, desired, actual, position=None): deletions = get_deletions(last_applied, desired) - delta = get_delta(last_applied, actual, desired, desired['kind']) + delta = get_delta(last_applied, actual, desired, position or desired['kind']) return dict_merge(deletions, delta) @@ -176,9 +176,8 @@ def list_merge(last_applied, actual, desired, position): if key not in actual_dict or key not in last_applied_dict: result.append(desired_dict[key]) else: - deletions = set(last_applied_dict[key].keys()) - set(desired_dict[key].keys()) - result.append(dict_merge({k: v for k, v in actual_dict[key].items() if k not in deletions}, - desired_dict[key])) + patch = merge(last_applied_dict[key], desired_dict[key], actual_dict[key], position) + result.append(dict_merge(actual_dict[key], patch)) return result else: return desired diff --git a/test/unit/test_apply.py b/test/unit/test_apply.py index a6cf0417..d8d74bb0 100644 --- a/test/unit/test_apply.py +++ b/test/unit/test_apply.py @@ -141,7 +141,29 @@ metadata=dict(name="foo"), spec=dict(ports=[dict(port=8443, name="https")]) ), - expected = dict(spec=dict(ports=[dict(port=8443, name="https", protocol='TCP')])) + expected = dict(spec=dict(ports=[dict(madeup=None, port=8443, name="https", protocol='TCP')])) + ), + dict( + last_applied = dict( + kind="Pod", + metadata=dict(name="foo"), + spec=dict(containers=[dict(name="busybox", image="busybox", + resources=dict(requests=dict(cpu="100m", memory="100Mi"), limits=dict(cpu="100m", memory="100Mi")))]) + ), + actual = dict( + kind="Pod", + metadata=dict(name="foo"), + spec=dict(containers=[dict(name="busybox", image="busybox", + resources=dict(requests=dict(cpu="100m", memory="100Mi"), limits=dict(cpu="100m", memory="100Mi")))]) + ), + desired = dict( + kind="Pod", + metadata=dict(name="foo"), + spec=dict(containers=[dict(name="busybox", image="busybox", + resources=dict(requests=dict(cpu="50m", memory="50Mi"), limits=dict(memory="50Mi")))]) + ), + expected=dict(spec=dict(containers=[dict(name="busybox", image="busybox", + resources=dict(requests=dict(cpu="50m", memory="50Mi"), limits=dict(cpu=None, memory="50Mi")))])) ), # This next one is based on a real world case where definition was mostly From 9e7a09e0679f4629228f32c27c2a5cf099f6836f Mon Sep 17 00:00:00 2001 From: OpenShift Cherrypick Robot Date: Fri, 15 May 2020 18:55:43 +0200 Subject: [PATCH 4/8] [release-0.11] Handle dryRun, and pass that on as a query param. (#363) * Handle dryRun, and pass that on as a query param. Co-authored-by: Mridul Gogoi --- openshift/dynamic/client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openshift/dynamic/client.py b/openshift/dynamic/client.py index 04da6360..000af480 100644 --- a/openshift/dynamic/client.py +++ b/openshift/dynamic/client.py @@ -210,6 +210,8 @@ def request(self, method, path, body=None, **params): query_params.append(('timeoutSeconds', params['timeout_seconds'])) if params.get('watch') is not None: query_params.append(('watch', params['watch'])) + if params.get('dry_run') is not None: + query_params.append(('dryRun', params['dry_run'])) header_params = params.get('header_params', {}) form_params = [] From 06268d83f4d43a6111a4ba027f520e5bcd96d581 Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Wed, 20 May 2020 15:59:23 -0400 Subject: [PATCH 5/8] Bump version --- openshift/__init__.py | 2 +- python-openshift.spec | 2 +- requirements.txt | 2 +- scripts/constants.py | 2 +- setup.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openshift/__init__.py b/openshift/__init__.py index a7eb8f9f..0333c745 100644 --- a/openshift/__init__.py +++ b/openshift/__init__.py @@ -14,5 +14,5 @@ # Do not edit these constants. They will be updated automatically # by scripts/update-version.sh. -__version__ = "0.11.0" +__version__ = "0.11.1" __k8s_client_version__ = "11.0.0" diff --git a/python-openshift.spec b/python-openshift.spec index 32980f4a..cdc15eeb 100644 --- a/python-openshift.spec +++ b/python-openshift.spec @@ -16,7 +16,7 @@ %endif Name: python-%{library} -Version: 0.11.0 +Version: 0.11.1 Release: 1%{?dist} Summary: Python client for the OpenShift API License: ASL 2.0 diff --git a/requirements.txt b/requirements.txt index 7147b662..9cef5704 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ dictdiffer jinja2 -kubernetes ~= 11.0 +kubernetes ~= 11.0.0 python-string-utils ruamel.yaml >= 0.15 six diff --git a/scripts/constants.py b/scripts/constants.py index 43dddb41..426ea96b 100644 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -23,7 +23,7 @@ # client version for packaging and releasing. It can # be different than SPEC_VERSION. -CLIENT_VERSION = "0.11.0" +CLIENT_VERSION = "0.11.1" KUBERNETES_CLIENT_VERSION = "11.0.0" # Name of the release package diff --git a/setup.py b/setup.py index 7ca5d897..f49ca27c 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ # Do not edit these constants. They will be updated automatically # by scripts/update-client.sh. -CLIENT_VERSION = "0.11.0" +CLIENT_VERSION = "0.11.1" PACKAGE_NAME = "openshift" DEVELOPMENT_STATUS = "3 - Alpha" From 43a072dd17ac46254fb3ff91c930b6e0508c921f Mon Sep 17 00:00:00 2001 From: OpenShift Cherrypick Robot Date: Sat, 6 Jun 2020 17:03:17 +0200 Subject: [PATCH 6/8] [release-0.11] Recursive diff (#370) * Fix strategic patch merge of env variables The key is `env`, not `envVars` * Add a Kubernetes specific recursive dict diff Understands strategic patch keys and can therefore better diff list elements * Remove unused dictdiffer dependency Co-authored-by: Will Thames --- openshift/dynamic/apply.py | 62 ++++++++++++++++++++++++++++++++-- python-openshift.spec | 4 +-- requirements.txt | 1 - test-requirements.txt | 1 - test/unit/test_diff.py | 69 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 test/unit/test_diff.py diff --git a/openshift/dynamic/apply.py b/openshift/dynamic/apply.py index 3b0ef325..3b4fb81d 100644 --- a/openshift/dynamic/apply.py +++ b/openshift/dynamic/apply.py @@ -15,15 +15,15 @@ 'imagePullSecrets': 'name', 'containers.volumeMounts': 'mountPath', 'containers.volumeDevices': 'devicePath', - 'containers.envVars': 'name', + 'containers.env': 'name', 'containers.ports': 'containerPort', 'initContainers.volumeMounts': 'mountPath', 'initContainers.volumeDevices': 'devicePath', - 'initContainers.envVars': 'name', + 'initContainers.env': 'name', 'initContainers.ports': 'containerPort', 'ephemeralContainers.volumeMounts': 'mountPath', 'ephemeralContainers.volumeDevices': 'devicePath', - 'ephemeralContainers.envVars': 'name', + 'ephemeralContainers.env': 'name', 'ephemeralContainers.ports': 'containerPort', } @@ -183,6 +183,62 @@ def list_merge(last_applied, actual, desired, position): return desired +def recursive_list_diff(list1, list2, position=None): + result = (list(), list()) + if position in STRATEGIC_MERGE_PATCH_KEYS: + patch_merge_key = STRATEGIC_MERGE_PATCH_KEYS[position] + dict1 = list_to_dict(list1, patch_merge_key, position) + dict2 = list_to_dict(list2, patch_merge_key, position) + dict1_keys = set(dict1.keys()) + dict2_keys = set(dict2.keys()) + for key in dict1_keys - dict2_keys: + result[0].append(dict1[key]) + for key in dict2_keys - dict1_keys: + result[1].append(dict2[key]) + for key in dict1_keys & dict2_keys: + diff = recursive_diff(dict1[key], dict2[key], position) + if diff: + # reinsert patch merge key to relate changes in other keys to + # a specific list element + diff[0].update({patch_merge_key: dict1[key][patch_merge_key]}) + diff[1].update({patch_merge_key: dict2[key][patch_merge_key]}) + result[0].append(diff[0]) + result[1].append(diff[1]) + if result[0] or result[1]: + return result + elif list1 != list2: + return (list1, list2) + return None + + +def recursive_diff(dict1, dict2, position=None): + if not position: + if 'kind' in dict1 and dict1.get('kind') == dict2.get('kind'): + position = dict1['kind'] + left = dict((k, v) for (k, v) in dict1.items() if k not in dict2) + right = dict((k, v) for (k, v) in dict2.items() if k not in dict1) + for k in (set(dict1.keys()) & set(dict2.keys())): + if position: + this_position = "%s.%s" % (position, k) + if isinstance(dict1[k], dict) and isinstance(dict2[k], dict): + result = recursive_diff(dict1[k], dict2[k], this_position) + if result: + left[k] = result[0] + right[k] = result[1] + elif isinstance(dict1[k], list) and isinstance(dict2[k], list): + result = recursive_list_diff(dict1[k], dict2[k], this_position) + if result: + left[k] = result[0] + right[k] = result[1] + elif dict1[k] != dict2[k]: + left[k] = dict1[k] + right[k] = dict2[k] + if left or right: + return left, right + else: + return None + + def get_deletions(last_applied, desired): patch = {} for k, last_applied_value in last_applied.items(): diff --git a/python-openshift.spec b/python-openshift.spec index cdc15eeb..17f2e93c 100644 --- a/python-openshift.spec +++ b/python-openshift.spec @@ -35,7 +35,6 @@ BuildRequires: python-setuptools BuildRequires: git Requires: python2 -Requires: python2-dictdiffer Requires: python2-kubernetes Requires: python2-string_utils Requires: python-requests @@ -55,7 +54,6 @@ BuildRequires: %{py3}-setuptools BuildRequires: git Requires: %{py3} -Requires: %{py3}-dictdiffer Requires: %{py3}-kubernetes Requires: %{py3}-string_utils Requires: %{py3}-requests @@ -92,7 +90,7 @@ Python client for the OpenShift API #the requirements are also done in an non-backwards compatible way %if 0%{?rhel} sed -i -e "s/find_packages(include='openshift.*')/['openshift', 'openshift.dynamic', 'openshift.helper']/g" setup.py -sed -i -e '30s/^/REQUIRES = [\n "dictdiffer",\n "jinja2",\n "kubernetes",\n "setuptools",\n "six",\n "ruamel.yaml",\n "python-string-utils",\n]\n/g' setup.py +sed -i -e '30s/^/REQUIRES = [\n "jinja2",\n "kubernetes",\n "setuptools",\n "six",\n "ruamel.yaml",\n "python-string-utils",\n]\n/g' setup.py sed -i -e "s/extract_requirements('requirements.txt')/REQUIRES/g" setup.py #sed -i -e '14,21d' setup.py %endif diff --git a/requirements.txt b/requirements.txt index 9cef5704..5fa256fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -dictdiffer jinja2 kubernetes ~= 11.0.0 python-string-utils diff --git a/test-requirements.txt b/test-requirements.txt index 0f7331e6..993602ad 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,4 +5,3 @@ pytest pytest-bdd pytest-cov PyYAML -dictdiffer diff --git a/test/unit/test_diff.py b/test/unit/test_diff.py new file mode 100644 index 00000000..a9b7c3fa --- /dev/null +++ b/test/unit/test_diff.py @@ -0,0 +1,69 @@ +from openshift.dynamic.apply import recursive_diff + +tests = [ + dict( + before = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8080, name="http")]) + ), + after = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8080, name="http")]) + ), + expected = None + ), + dict( + before = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8080, name="http")]) + ), + after = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8081, name="http")]) + ), + expected = ( + dict(spec=dict(ports=[dict(port=8080, name="http")])), + dict(spec=dict(ports=[dict(port=8081, name="http")])) + ) + ), + dict( + before = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8080, name="http"), dict(port=8081, name="https")]) + ), + after = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8081, name="https"), dict(port=8080, name="http")]) + ), + expected = None + ), + + dict( + before = dict( + kind="Pod", + metadata=dict(name="foo"), + spec=dict(containers=[dict(name="busybox", image="busybox", + env=[dict(name="hello", value="world"), + dict(name="another", value="next")])]) + ), + after = dict( + kind="Pod", + metadata=dict(name="foo"), + spec=dict(containers=[dict(name="busybox", image="busybox", + env=[dict(name="hello", value="everyone")])]) + ), + expected=(dict(spec=dict(containers=[dict(name="busybox", env=[dict(name="another", value="next"), dict(name="hello", value="world")])])), + dict(spec=dict(containers=[dict(name="busybox", env=[dict(name="hello", value="everyone")])]))) + ), + ] + + +def test_diff(): + for test in tests: + assert(recursive_diff(test['before'], test['after']) == test['expected']) From e2e6989e71d6bb6ed578906a39f074ac3b1e2347 Mon Sep 17 00:00:00 2001 From: OpenShift Cherrypick Robot Date: Sat, 6 Jun 2020 17:03:24 +0200 Subject: [PATCH 7/8] Handle list merges where items have been added elsewhere (#371) For example, a Pod has its service account token mounted as a volume - we don't want to later try and remove it. Other examples are likely available Co-authored-by: Will Thames --- openshift/dynamic/apply.py | 3 +++ test/unit/test_apply.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/openshift/dynamic/apply.py b/openshift/dynamic/apply.py index 3b4fb81d..ed42be11 100644 --- a/openshift/dynamic/apply.py +++ b/openshift/dynamic/apply.py @@ -178,6 +178,9 @@ def list_merge(last_applied, actual, desired, position): else: patch = merge(last_applied_dict[key], desired_dict[key], actual_dict[key], position) result.append(dict_merge(actual_dict[key], patch)) + for key in actual_dict: + if key not in desired_dict and key not in last_applied_dict: + result.append(actual_dict[key]) return result else: return desired diff --git a/test/unit/test_apply.py b/test/unit/test_apply.py index d8d74bb0..907d08d2 100644 --- a/test/unit/test_apply.py +++ b/test/unit/test_apply.py @@ -53,6 +53,7 @@ ), expected = dict(metadata=dict(annotations=None), data=dict(two=None, three="3")) ), + dict( last_applied = dict( kind="Service", @@ -165,6 +166,43 @@ expected=dict(spec=dict(containers=[dict(name="busybox", image="busybox", resources=dict(requests=dict(cpu="50m", memory="50Mi"), limits=dict(cpu=None, memory="50Mi")))])) ), + dict( + desired = dict(kind='Pod', + spec=dict(containers=[ + dict(name='hello', + volumeMounts=[dict(name="test", mountPath="/test")]) + ], + volumes=[ + dict(name="test", configMap=dict(name="test")), + ])), + last_applied = dict(kind='Pod', + spec=dict(containers=[ + dict(name='hello', + volumeMounts=[dict(name="test", mountPath="/test")]) + ], + volumes=[ + dict(name="test", configMap=dict(name="test")), + ])), + actual = dict(kind='Pod', + spec=dict(containers=[ + dict(name='hello', + volumeMounts=[dict(name="test", mountPath="/test"), + dict(mountPath="/var/run/secrets/kubernetes.io/serviceaccount", name="default-token-xyz")]) + ], + volumes=[ + dict(name="test", configMap=dict(name="test")), + dict(name="default-token-xyz", secret=dict(secretName="default-token-xyz")), + ])), + expected = dict(spec=dict(containers=[ + dict(name='hello', + volumeMounts=[dict(name="test", mountPath="/test"), + dict(mountPath="/var/run/secrets/kubernetes.io/serviceaccount", name="default-token-xyz")]) + ], + volumes=[ + dict(name="test", configMap=dict(name="test")), + dict(name="default-token-xyz", secret=dict(secretName="default-token-xyz")), + ])), + ), # This next one is based on a real world case where definition was mostly # str type and everything else was mostly unicode type (don't ask me how) From f5c66d2ffa60edfac829a6cc7c94661de0b46b7d Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Sat, 6 Jun 2020 11:07:56 -0400 Subject: [PATCH 8/8] Bump version --- openshift/__init__.py | 2 +- python-openshift.spec | 2 +- scripts/constants.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openshift/__init__.py b/openshift/__init__.py index 0333c745..ae4e3874 100644 --- a/openshift/__init__.py +++ b/openshift/__init__.py @@ -14,5 +14,5 @@ # Do not edit these constants. They will be updated automatically # by scripts/update-version.sh. -__version__ = "0.11.1" +__version__ = "0.11.2" __k8s_client_version__ = "11.0.0" diff --git a/python-openshift.spec b/python-openshift.spec index 17f2e93c..43e148c1 100644 --- a/python-openshift.spec +++ b/python-openshift.spec @@ -16,7 +16,7 @@ %endif Name: python-%{library} -Version: 0.11.1 +Version: 0.11.2 Release: 1%{?dist} Summary: Python client for the OpenShift API License: ASL 2.0 diff --git a/scripts/constants.py b/scripts/constants.py index 426ea96b..9c3c3b5c 100644 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -23,7 +23,7 @@ # client version for packaging and releasing. It can # be different than SPEC_VERSION. -CLIENT_VERSION = "0.11.1" +CLIENT_VERSION = "0.11.2" KUBERNETES_CLIENT_VERSION = "11.0.0" # Name of the release package diff --git a/setup.py b/setup.py index f49ca27c..4c7d8dc8 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ # Do not edit these constants. They will be updated automatically # by scripts/update-client.sh. -CLIENT_VERSION = "0.11.1" +CLIENT_VERSION = "0.11.2" PACKAGE_NAME = "openshift" DEVELOPMENT_STATUS = "3 - Alpha"