From 5217c3182cf89ca173ae8078b7a39a05743c663f Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 21:57:47 +0200 Subject: [PATCH 01/11] chore: Update version in antora.yml to 25.7.0 --- antora.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/antora.yml b/antora.yml index 3d9439389..274dd8114 100644 --- a/antora.yml +++ b/antora.yml @@ -1,5 +1,5 @@ name: home -version: "nightly" +version: "25.7" title: Stackable Documentation nav: - modules/ROOT/nav1.adoc @@ -14,7 +14,7 @@ nav: - modules/ROOT/nav3.adoc # this is for the extra bits at the end of the menu # The prerelease setting affects version sorting. # Set to 'true' for nightly and false otherwise. -prerelease: true +prerelease: false # The attributes below are specific to this component and version # https://docs.antora.org/antora/latest/component-attributes/#hard-set asciidoc: @@ -22,7 +22,7 @@ asciidoc: # Keep this version in line with the 'version' key above # The versions for the CRD docs are either 'nightly' or # a full major.minor.patch version like 23.7.1 - crd-docs-version: "nightly" + crd-docs-version: "25.7.0" # Whether this version is already end of life. # If true, a banner will be displayed informing the user. end-of-life: false From d6a9c5c7726770a4475cddf815934b2c52f49197 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 25 Jul 2025 09:12:28 +0200 Subject: [PATCH 02/11] chore: Adjust branch names for 25.7 --- antora-playbook.yml | 34 +++++++++++++++++----------------- local-antora-playbook.yml | 34 +++++++++++++++++----------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/antora-playbook.yml b/antora-playbook.yml index 067e14cd0..7cddaa598 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -25,7 +25,7 @@ content: sources: - url: . branches: - - HEAD + - release-25.7 - release/25.3 - release/24.11 - release/24.7 @@ -42,7 +42,7 @@ content: - url: https://github.com/stackabletech/demos.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -55,7 +55,7 @@ content: - url: https://github.com/stackabletech/commons-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -67,7 +67,7 @@ content: - url: https://github.com/stackabletech/secret-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -79,7 +79,7 @@ content: - url: https://github.com/stackabletech/listener-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -92,7 +92,7 @@ content: - url: https://github.com/stackabletech/airflow-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -104,7 +104,7 @@ content: - url: https://github.com/stackabletech/druid-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -116,7 +116,7 @@ content: - url: https://github.com/stackabletech/hbase-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -128,7 +128,7 @@ content: - url: https://github.com/stackabletech/hdfs-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -140,7 +140,7 @@ content: - url: https://github.com/stackabletech/hive-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -152,7 +152,7 @@ content: - url: https://github.com/stackabletech/kafka-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -164,7 +164,7 @@ content: - url: https://github.com/stackabletech/nifi-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -176,7 +176,7 @@ content: - url: https://github.com/stackabletech/opa-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -192,7 +192,7 @@ content: - url: https://github.com/stackabletech/spark-k8s-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -204,7 +204,7 @@ content: - url: https://github.com/stackabletech/superset-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -216,7 +216,7 @@ content: - url: https://github.com/stackabletech/trino-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -228,7 +228,7 @@ content: - url: https://github.com/stackabletech/zookeeper-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index 21b9ac69f..787603c96 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -14,7 +14,7 @@ content: sources: - url: ./ branches: - - HEAD + - release-25.7 - release/25.3 - release/24.11 - release/24.7 @@ -31,7 +31,7 @@ content: - url: https://github.com/stackabletech/demos.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -44,7 +44,7 @@ content: - url: https://github.com/stackabletech/commons-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -56,7 +56,7 @@ content: - url: https://github.com/stackabletech/secret-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -68,7 +68,7 @@ content: - url: https://github.com/stackabletech/listener-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -81,7 +81,7 @@ content: - url: https://github.com/stackabletech/airflow-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -93,7 +93,7 @@ content: - url: https://github.com/stackabletech/druid-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -105,7 +105,7 @@ content: - url: https://github.com/stackabletech/hbase-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -117,7 +117,7 @@ content: - url: https://github.com/stackabletech/hdfs-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -129,7 +129,7 @@ content: - url: https://github.com/stackabletech/hive-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -141,7 +141,7 @@ content: - url: https://github.com/stackabletech/kafka-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -153,7 +153,7 @@ content: - url: https://github.com/stackabletech/nifi-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -165,7 +165,7 @@ content: - url: https://github.com/stackabletech/opa-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -181,7 +181,7 @@ content: - url: https://github.com/stackabletech/spark-k8s-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -193,7 +193,7 @@ content: - url: https://github.com/stackabletech/superset-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -205,7 +205,7 @@ content: - url: https://github.com/stackabletech/trino-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 @@ -217,7 +217,7 @@ content: - url: https://github.com/stackabletech/zookeeper-operator.git start_path: docs branches: - - main + - release-25.7 - release-25.3 - release-24.11 - release-24.7 From dcd334d3503832cc308b8a8093cd2cf1b9a7d578 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 25 Jul 2025 10:10:36 +0200 Subject: [PATCH 03/11] Revert "Hide OpenSearch on released versions (#759)" This reverts commit 59ec22c18f2f4db73f84293b9574564bba4a443e. --- supplemental-ui/partials/navbar.hbs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/supplemental-ui/partials/navbar.hbs b/supplemental-ui/partials/navbar.hbs index 81cd2d2a8..10817938f 100644 --- a/supplemental-ui/partials/navbar.hbs +++ b/supplemental-ui/partials/navbar.hbs @@ -37,19 +37,7 @@ Apache Hive Apache Kafka Apache NiFi - {{#if (not (or - (eq page.version "23.1") - (eq page.version "23.4") - (eq page.version "23.7") - (eq page.version "23.11") - (eq page.version "24.3") - (eq page.version "24.7") - (eq page.version "24.11") - (eq page.version "25.3") - (eq page.version "25.7") - ))}} OpenSearch - {{/if}} Apache Spark on K8S Apache Superset Trino From fc1eacac9b9d925ddf8f27e70f1973d506aa97be Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 25 Jul 2025 10:10:49 +0200 Subject: [PATCH 04/11] Revert "Add opensearch-operator (#758)" This reverts commit 1d1d1a8a4f354fe74c877a57317a0e166d0c26e5. --- antora-playbook.yml | 4 ---- local-antora-playbook.yml | 4 ---- modules/ROOT/pages/product-information.adoc | 1 - .../pages/stackable_resource_requests.adoc | 2 +- modules/operators/pages/index.adoc | 18 +----------------- .../operators/pages/supported_versions.adoc | 4 ---- .../operators/partials/operator_doc_links.adoc | 5 ----- modules/tutorials/pages/index.adoc | 1 - only-dev-antora-playbook.yml | 4 ---- supplemental-ui/partials/navbar.hbs | 1 - truly-local-playbook.yml | 2 -- 11 files changed, 2 insertions(+), 44 deletions(-) diff --git a/antora-playbook.yml b/antora-playbook.yml index 7cddaa598..7f9cadf39 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -185,10 +185,6 @@ content: - release-23.7 - release-23.4 - release-23.1 - - url: https://github.com/stackabletech/opensearch-operator.git - start_path: docs - branches: - - main - url: https://github.com/stackabletech/spark-k8s-operator.git start_path: docs branches: diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index 787603c96..dce9f7c78 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -174,10 +174,6 @@ content: - release-23.7 - release-23.4 - release-23.1 - - url: https://github.com/stackabletech/opensearch-operator.git - start_path: docs - branches: - - main - url: https://github.com/stackabletech/spark-k8s-operator.git start_path: docs branches: diff --git a/modules/ROOT/pages/product-information.adoc b/modules/ROOT/pages/product-information.adoc index 24bc26cd8..6e95ed14b 100644 --- a/modules/ROOT/pages/product-information.adoc +++ b/modules/ROOT/pages/product-information.adoc @@ -28,7 +28,6 @@ Supported products: * xref:hdfs:index.adoc[Apache Hadoop HDFS] * xref:kafka:index.adoc[Apache Kafka] * xref:nifi:index.adoc[Apache Nifi] -* xref:opensearch:index.adoc[OpenSearch] * xref:spark-k8s:index.adoc[Apache Spark] (including xref:spark-k8s:usage-guide/history-server.adoc[Spark History Server]) * xref:superset:index.adoc[Apache Superset] * xref:trino:index.adoc[Trino] diff --git a/modules/concepts/pages/stackable_resource_requests.adoc b/modules/concepts/pages/stackable_resource_requests.adoc index b51a89052..b6a66c85e 100644 --- a/modules/concepts/pages/stackable_resource_requests.adoc +++ b/modules/concepts/pages/stackable_resource_requests.adoc @@ -1,7 +1,7 @@ // This is meant to be inlined using the "include" directive in other pages. // WARNING: do not add headers here as they can break the structure of pages // that include this file. -Stackable operators handle resource requests in a slightly different manner than Kubernetes. +Stackable operators handle resource requests in a sligtly different manner than Kubernetes. Resource requests are defined on xref:concepts:stacklet.adoc#roles[role] or xref:concepts:stacklet.adoc#role-groups[role group] level. On a role level this means that by default, all workers will use the same resource requests and limits. This can be further specified on role group level (which takes priority to the role level) to apply different resources. diff --git a/modules/operators/pages/index.adoc b/modules/operators/pages/index.adoc index 3e7c8b15e..ef267e83e 100644 --- a/modules/operators/pages/index.adoc +++ b/modules/operators/pages/index.adoc @@ -138,22 +138,6 @@ xref:nifi:index.adoc[Read more]
++++ -++++ -

OpenSearch

-++++ - -OpenSearch is a powerful search and analytics engine built on Apache Lucene. - -xref:opensearch:index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - ++++

Apache Spark

++++ @@ -299,4 +283,4 @@ xref:listener-operator:index.adoc[Read more] ++++
-++++ +++++ \ No newline at end of file diff --git a/modules/operators/pages/supported_versions.adoc b/modules/operators/pages/supported_versions.adoc index 82da02f3b..b575a88ed 100644 --- a/modules/operators/pages/supported_versions.adoc +++ b/modules/operators/pages/supported_versions.adoc @@ -38,10 +38,6 @@ include::nifi:partial$supported-versions.adoc[] include::opa:partial$supported-versions.adoc[] -== OpenSearch - -include::opensearch:partial$supported-versions.adoc[] - == Apache Spark on Kubernetes include::spark-k8s:partial$supported-versions.adoc[] diff --git a/modules/operators/partials/operator_doc_links.adoc b/modules/operators/partials/operator_doc_links.adoc index 3c81078ff..f835b8487 100644 --- a/modules/operators/partials/operator_doc_links.adoc +++ b/modules/operators/partials/operator_doc_links.adoc @@ -34,11 +34,6 @@ include::kafka:partial$nav.adoc[] -- include::nifi:partial$nav.adoc[] -- -** xref:opensearch:index.adoc[OpenSearch] -+ --- -include::opensearch:partial$nav.adoc[] --- ** xref:spark-k8s:index.adoc[Apache Spark on K8S] + -- diff --git a/modules/tutorials/pages/index.adoc b/modules/tutorials/pages/index.adoc index dc8f45e49..e3a563cc0 100644 --- a/modules/tutorials/pages/index.adoc +++ b/modules/tutorials/pages/index.adoc @@ -21,7 +21,6 @@ Follow any of these guides to get started with a specific product. * xref:kafka:getting_started/index.adoc[Stackable Operator for Apache Kafka] * xref:nifi:getting_started/index.adoc[Stackable Operator for Apache NiFi] * xref:opa:getting_started/index.adoc[Stackable Operator for OpenPolicyAgent] -* xref:opensearch:getting_started/index.adoc[Stackable Operator for OpenSearch] * xref:spark-k8s:getting_started/index.adoc[Stackable Operator for Apache Spark] * xref:superset:getting_started/index.adoc[Stackable Operator for Apache Superset] * xref:trino:getting_started/index.adoc[Stackable Operator for Trino] diff --git a/only-dev-antora-playbook.yml b/only-dev-antora-playbook.yml index bd6236c1e..dbc5d13b5 100644 --- a/only-dev-antora-playbook.yml +++ b/only-dev-antora-playbook.yml @@ -70,10 +70,6 @@ content: start_path: docs branches: - main - - url: https://github.com/stackabletech/opensearch-operator.git - start_path: docs - branches: - - main - url: https://github.com/stackabletech/spark-k8s-operator.git start_path: docs branches: diff --git a/supplemental-ui/partials/navbar.hbs b/supplemental-ui/partials/navbar.hbs index 10817938f..72fd2bb0a 100644 --- a/supplemental-ui/partials/navbar.hbs +++ b/supplemental-ui/partials/navbar.hbs @@ -37,7 +37,6 @@ Apache Hive Apache Kafka Apache NiFi - OpenSearch Apache Spark on K8S Apache Superset Trino diff --git a/truly-local-playbook.yml b/truly-local-playbook.yml index 78b43ca04..21675d785 100644 --- a/truly-local-playbook.yml +++ b/truly-local-playbook.yml @@ -45,8 +45,6 @@ content: start_path: docs - url: ../opa-operator/ start_path: docs - - url: ../opensearch-operator/ - start_path: docs - url: ../spark-k8s-operator/ start_path: docs - url: ../superset-operator/ From f017d1aac97c0f448aed1f06bbeb0480e2d02c1b Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 25 Jul 2025 10:11:13 +0200 Subject: [PATCH 05/11] chore: Fix typo --- modules/concepts/pages/stackable_resource_requests.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/concepts/pages/stackable_resource_requests.adoc b/modules/concepts/pages/stackable_resource_requests.adoc index b6a66c85e..b51a89052 100644 --- a/modules/concepts/pages/stackable_resource_requests.adoc +++ b/modules/concepts/pages/stackable_resource_requests.adoc @@ -1,7 +1,7 @@ // This is meant to be inlined using the "include" directive in other pages. // WARNING: do not add headers here as they can break the structure of pages // that include this file. -Stackable operators handle resource requests in a sligtly different manner than Kubernetes. +Stackable operators handle resource requests in a slightly different manner than Kubernetes. Resource requests are defined on xref:concepts:stacklet.adoc#roles[role] or xref:concepts:stacklet.adoc#role-groups[role group] level. On a role level this means that by default, all workers will use the same resource requests and limits. This can be further specified on role group level (which takes priority to the role level) to apply different resources. From f27f0350d2c9686e616553bd1cf35d4e4a0d7d20 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 25 Jul 2025 10:19:56 +0200 Subject: [PATCH 06/11] chore: Make pre-commit happy --- modules/operators/pages/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/operators/pages/index.adoc b/modules/operators/pages/index.adoc index ef267e83e..46db634fa 100644 --- a/modules/operators/pages/index.adoc +++ b/modules/operators/pages/index.adoc @@ -283,4 +283,4 @@ xref:listener-operator:index.adoc[Read more] ++++ -++++ \ No newline at end of file +++++ From 2311690a8daa343cfa0aa424de2577ce586b04cc Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 25 Jul 2025 13:24:56 +0200 Subject: [PATCH 07/11] chore: Update Service exposition concepts page (backport) (#767) --- .../concepts/pages/service-exposition.adoc | 85 +++++++++---------- 1 file changed, 39 insertions(+), 46 deletions(-) diff --git a/modules/concepts/pages/service-exposition.adoc b/modules/concepts/pages/service-exposition.adoc index 0c0f8e86f..f2025d04b 100644 --- a/modules/concepts/pages/service-exposition.adoc +++ b/modules/concepts/pages/service-exposition.adoc @@ -1,70 +1,63 @@ = Service exposition -:k8s-service: https://kubernetes.io/docs/concepts/services-networking/service/ -:k8s-service-types: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types -:description: Explore Stackable's service exposition options: ClusterIP for internal access, NodePort for unstable external access, and LoadBalancer for stable external access. - +:listener-operator: xref:listener-operator:index.adoc +:secret-operator: xref:secret-operator:index.adoc +:listenerclass: xref:listener-operator:listenerclass.adoc +:description: Explore how Stackable utilizes the listener-operator to expose Services. Data products expose interfaces to the outside world. These interfaces (whether UIs, or APIs) can be accessed by other products or by end users. -Other products accessing the interfaces can run inside or outside of the same Kubernetes cluster. +Clients accessing the interfaces can run inside or outside of the same Kubernetes cluster. For example, xref:zookeeper:index.adoc[Apache ZooKeeper] is a dependency for other products, and it usually needs to be accessible only from within Kubernetes, while xref:superset:index.adoc[Apache Superset] is a data analysis product for end users and therefore needs to be accessible from outside the Kubernetes cluster. Users connecting to Superset can be restricted within the local company network, or they can connect over the internet depending on the company security policies and demands. This page gives an overview over the different options for service exposition, when to choose which option and how these options are configured. -== Service exposition options - -The Stackable Data Platform supports three {k8s-service-types}[types of Kubernetes Service] for exposing data product endpoints: +== Motivation -* ClusterIP -* NodePort -* LoadBalancer +Service exposition is such a complicated topic, that Stackable has build it's own operator for that: {listener-operator}[]. +The following section explains the motivation behind implementing such an operator instead of using plain regular Kubernetes Services. -All custom resources for data products provide a resource field named `spec.clusterConfig.listenerClass` which determines how the product can be accessed. -There are three ListenerClasses, named after the goal for which they are used (more on this in the <>): +=== Products advertising their addresses -* `cluster-internal` => Use ClusterIP (default) -* `external-unstable` => Use NodePort -* `external-stable` => Use LoadBalancer +Some products require information about their external accessibility. +This is e.g. important for HDFS, where the namenode keeps track of which datanode serves which block. Another case is Kafka, where it is required for client bootstrapping. +A common use case is an HDFS client connecting to a namenode in order to read block 42. Therefore, the namenode needs to know which datanode is serving block 42. The namenode then responds with the IP or hostname of the datanode containing that block 42. +For that to work, the datanode needs to know it's external address on startup and tell it the namenode. +(And yes, we needed to patch the Hadoop sourcecode for that) -The `cluster-internal` class exposes the interface of a product by using a ClusterIP Service. -This service is only reachable from within the Kubernetes cluster. -This setting is the most secure and was chosen as the default for that reason. +The {listener-operator}[listener-operator] runs as CSI driver (same as the {secret-operator}[secret-operator]) and places files inside the CSI volume, which tell the tool how it is reachable. -NOTE: Not all operators support all classes. -Consult the operator specific documentation to find out about the supported service types. +=== Integration with {secret-operator}[secret-operator] -[#when-to-choose-which-option] -== When to choose which option +If a tool is secured using TLS or Kerberos, it does not only need to be reachable via the determined address, it also needs a TLS certificate/keytab issued on the determined address. +{secret-operator}[secret-operator] integrated with to {listener-operator}[listener-operator], so that the platform takes care of provisioning certificates with the correct addresses (in the form of SAN entries). -There are three options, one for internal traffic and two for external access, where internal and external refer to the Kubernetes cluster. -Internal means inside of the Kuberenetes cluster, and external means access from outside of it. +== {listenerclass}[ListenerClasses] -=== Internal +A {listenerclass}[] describes how a product should be exposed. +Please read on {listenerclass}[its documentation] before continuing on this page. -`cluster-internal` is the default class and the Service behind it is only reachable from within Kubernetes. -This is useful for middleware products such as xref:zookeeper:index.adoc[Apache ZooKeeper], xref:hive:index.adoc[Apache Hive metastore], or an xref:kafka:index.adoc[Apache Kafka] cluster used for internal data flow. -Products using this ListenerClass are not accessible from outside Kubernetes. +As a quick reminder, the platform ships with 3 default {listenerclass}[ListenerClasses]: -=== External +`cluster-internal`:: Used for listeners that are only accessible internally from the cluster. For example: communication between ZooKeeper nodes. +`external-unstable`:: Used for listeners that are accessible from outside the cluster, but which do not require a stable address. For example: individual Kafka brokers. +`external-stable`:: Used for listeners that are accessible from outside the cluster, and do require a stable address. For example: Kafka bootstrap. -External access is needed when a product needs to be accessed from _outside_ of Kubernetes. -This is necessary for all end user products such as xref:superset:index.adoc[Apache Superset]. -Some tools can expose APIs for data ingestion like xref:kafka:index.adoc[Apache Kafka] or xref:nifi:index.adoc[Apache NiFi]. -If data needs to be ingested from outside of the cluster, one of the external listener classes should be chosen. +Keep in mind that you are not restricted to this list, you can configure your own custom {listenerclass}[ListenerClasses]. -When to use `stable` and when to use `unstable`? -The `external-unstable` setting exposes a product interface via a Kuberneres NodePort. -In this case the service's IP address and port can change if Kubernetes needs to restart or reschedule the Pod to another node. +== Configuring the ListenerClass for a Stacklet -The `external-stable` class uses a LoadBalancer. -The LoadBalancer is running at a fixed address and is therefore `stable`. -Managed Kubernetes services in the cloud usually offer a LoadBalancer, but for an on premise cluster you have to configure a LoadBalancer yourself. -For a production setup, it is recommended to use a LoadBalancer and the `external-stable` ListenerClass. +The {listener-operator}[listener-operator] is integrated into most of the Stackable products, currently only xref:opa:index.adoc[] and xref:spark-k8s:index.adoc[] are not using {listener-operator}[listener-operator]. -== Outlook +Most of the products configure the {listenerclass}[] at the role level as follows. +However, there are some products that have this option at the rolegroup level. +One example is HDFS, where some roles require a listener service per Pod, to individually access single instances. -For most of the Stackable operators, these listener classes are hardcoded to expose certain Service types and do not offer any additional configuration. -However, some operators support specifying custom xref:listener-operator:listenerclass.adoc[ListenerClass]es with more granular configuration options, via the xref:listener-operator:index.adoc[listener-operator]. -In a future release, all Stackable operators are planned to be migrated over to this system. +[source,yaml] +---- +spec: + my-role: + roleConfig: + listenerClass: external-unstable +---- -For more information on what is supported by any individual operator, please see that operator's documentation. +Every operator has a documentation section called "Service exposition with ListenerClasses", which may provide details for the specific tool. From a48569b8e9889e4512624b9f8822bcd9f1f0acdc Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Tue, 5 Aug 2025 11:48:47 +0200 Subject: [PATCH 08/11] Add known issue on PDBs and certificate rotation (backport) --- .../pages/operations/pod_disruptions.adoc | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/modules/concepts/pages/operations/pod_disruptions.adoc b/modules/concepts/pages/operations/pod_disruptions.adoc index c9bbf561a..7d1da11ac 100644 --- a/modules/concepts/pages/operations/pod_disruptions.adoc +++ b/modules/concepts/pages/operations/pod_disruptions.adoc @@ -117,3 +117,37 @@ This PDB allows only one Pod out of all the Namenodes and Journalnodes to be dow == Details Have a look at the xref:contributor:adr/ADR030-allowed-pod-disruptions.adoc[ADR on Allowed Pod disruptions] for the implementation details. + +== Known issue with PDBs and certificate rotations +PDBs together with certificate rotations can be problematic in case e.g. {commons-operator}[commons-operator] was unavailable to restart the Pods before the certificate expire. +commons-operator uses the `evict` API in Kubernetes, which respects the PDB. +If a Pod is evicted and a PDB would be violated, the Pod is *not* restarted. +Assume a product like xref:zookeeper:index.adoc[Apache ZooKeeper] which needs to form a quorum to function and the PDB only allows a single Pod to be unavailable. +As soon as enough certificates of the ZookeeperCluster have expired, all Pods will crash-loop, as they encounter expired certificates. +As only the container crash-loops (not the entire Pod), no new certificate is issued. +As soon as commons-operator comes online again it tries to `evict` a Zookeeper Pod. +However, this is prohibited, as the PDB would be violated. + +NOTE: We encountered this problem only with the specific outlined case above and only under this circumstances. + +=== Workaround +If you encounter this only manually deleting those pods can help out of this situation. +A Pod deletion (other than evictions) does *not* respect PDBs, so the Pods can be restarted anyway. +All restarted Pods will get a new certificate, the stacklet should turn healthy again. + +=== Restore working state +Delete pods with e.g. `kubectl``. +[source, bash] +---- +kubectl delete pod -l app.kubernetes.io/name=zookeeper,app.kubernetes.io/instance=simple-zk +pod "simple-zk-server-default-0" deleted +pod "simple-zk-server-default-1" deleted +pod "simple-zk-server-default-2" deleted +---- + + +=== Preventing this situation +The best measure is to make sure that commons-operator is always running, so that it can restart the Pods before the certificates expire. + +A hacky way to prevent this situation could be to disable PDBs for the specific stacklet. +But this also has the downside, that you are now missing the benefits of the PDB. From 8f7cc3b5437667dad46d8ae412c963090fd80b19 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 8 Aug 2025 16:22:38 +0200 Subject: [PATCH 09/11] Revert "Add known issue on PDBs and certificate rotation (backport)" This reverts commit a48569b8e9889e4512624b9f8822bcd9f1f0acdc. --- .../pages/operations/pod_disruptions.adoc | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/modules/concepts/pages/operations/pod_disruptions.adoc b/modules/concepts/pages/operations/pod_disruptions.adoc index 7d1da11ac..c9bbf561a 100644 --- a/modules/concepts/pages/operations/pod_disruptions.adoc +++ b/modules/concepts/pages/operations/pod_disruptions.adoc @@ -117,37 +117,3 @@ This PDB allows only one Pod out of all the Namenodes and Journalnodes to be dow == Details Have a look at the xref:contributor:adr/ADR030-allowed-pod-disruptions.adoc[ADR on Allowed Pod disruptions] for the implementation details. - -== Known issue with PDBs and certificate rotations -PDBs together with certificate rotations can be problematic in case e.g. {commons-operator}[commons-operator] was unavailable to restart the Pods before the certificate expire. -commons-operator uses the `evict` API in Kubernetes, which respects the PDB. -If a Pod is evicted and a PDB would be violated, the Pod is *not* restarted. -Assume a product like xref:zookeeper:index.adoc[Apache ZooKeeper] which needs to form a quorum to function and the PDB only allows a single Pod to be unavailable. -As soon as enough certificates of the ZookeeperCluster have expired, all Pods will crash-loop, as they encounter expired certificates. -As only the container crash-loops (not the entire Pod), no new certificate is issued. -As soon as commons-operator comes online again it tries to `evict` a Zookeeper Pod. -However, this is prohibited, as the PDB would be violated. - -NOTE: We encountered this problem only with the specific outlined case above and only under this circumstances. - -=== Workaround -If you encounter this only manually deleting those pods can help out of this situation. -A Pod deletion (other than evictions) does *not* respect PDBs, so the Pods can be restarted anyway. -All restarted Pods will get a new certificate, the stacklet should turn healthy again. - -=== Restore working state -Delete pods with e.g. `kubectl``. -[source, bash] ----- -kubectl delete pod -l app.kubernetes.io/name=zookeeper,app.kubernetes.io/instance=simple-zk -pod "simple-zk-server-default-0" deleted -pod "simple-zk-server-default-1" deleted -pod "simple-zk-server-default-2" deleted ----- - - -=== Preventing this situation -The best measure is to make sure that commons-operator is always running, so that it can restart the Pods before the certificates expire. - -A hacky way to prevent this situation could be to disable PDBs for the specific stacklet. -But this also has the downside, that you are now missing the benefits of the PDB. From 63114f3fa70d23c14466a8fc6a557878ed637145 Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Fri, 8 Aug 2025 16:48:15 +0200 Subject: [PATCH 10/11] Add known issue on PDBs and certificate rotation (#769) (#772) * Add documentation about PDBs * Fix precommit * Requested Changes * Only k delte command rather then options * Apply formatting changes --------- Co-authored-by: Sebastian Bernauer --- .../pages/operations/pod_disruptions.adoc | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/modules/concepts/pages/operations/pod_disruptions.adoc b/modules/concepts/pages/operations/pod_disruptions.adoc index c9bbf561a..657986692 100644 --- a/modules/concepts/pages/operations/pod_disruptions.adoc +++ b/modules/concepts/pages/operations/pod_disruptions.adoc @@ -1,5 +1,6 @@ = Allowed Pod disruptions :k8s-pdb: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +:commons-operator: xref:commons-operator:index.adoc :description: Configure PodDisruptionBudgets (PDBs) to minimize planned downtime for Stackable products. Default values are based on fault tolerance and can be customized. Any downtime of our products is generally considered to be bad. @@ -117,3 +118,38 @@ This PDB allows only one Pod out of all the Namenodes and Journalnodes to be dow == Details Have a look at the xref:contributor:adr/ADR030-allowed-pod-disruptions.adoc[ADR on Allowed Pod disruptions] for the implementation details. + +== Known issue with PDBs and certificate rotations +PDBs together with certificate rotations can be problematic in case e.g. {commons-operator}[commons-operator] was unavailable to restart the Pods before the certificate expire. +commons-operator uses the `evict` API in Kubernetes, which respects the PDB. +If a Pod is evicted and a PDB would be violated, the Pod is *not* restarted. + +Assume a product like xref:zookeeper:index.adoc[Apache ZooKeeper] which needs to form a quorum to function and the PDB only allows a single Pod to be unavailable. +As soon as enough certificates of the ZookeeperCluster have expired, all Pods will crash-loop, as they encounter expired certificates. +As only the container crash-loops (not the entire Pod), no new certificate is issued. +As soon as commons-operator comes online again it tries to `evict` a Zookeeper Pod. +However, this is prohibited, as the PDB would be violated. + +NOTE: We encountered this problem only with the specific outlined case above and only under this circumstances. + +=== Workaround +If you encounter this, only manually deleting those pods can help you out of this situation. +A Pod deletion (other than evictions) does *not* respect PDBs, so the Pods can be restarted anyway. +All restarted Pods will get a new certificate, the stacklet should turn healthy again. + +=== Restore working state +Delete pods with e.g. `kubectl``. +[source, bash] +---- +kubectl delete pod -l app.kubernetes.io/name=zookeeper,app.kubernetes.io/instance=simple-zk +pod "simple-zk-server-default-0" deleted +pod "simple-zk-server-default-1" deleted +pod "simple-zk-server-default-2" deleted +---- + + +=== Preventing this situation +The best measure is to make sure that commons-operator is always running, so that it can restart the Pods before the certificates expire. + +A hacky way to prevent this situation could be to disable PDBs for the specific stacklet. +But this also has the downside, that you are now missing the benefits of the PDB. From 76fe9410fa1a5ed242d7f515f3588c204c72b0d2 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 22 Aug 2025 14:04:28 +0200 Subject: [PATCH 11/11] docs: Mention missed Trino PVC removal/fix in 25.7.0 release notes (#776) --- .../ROOT/partials/release-notes/release-25.7.adoc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 044fdd7a0..5409d88ab 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -259,6 +259,21 @@ The `-nodeport` discovery ConfigMap has been deprecated in 25.3.0 and is removed Use the primary discovery ConfigMap instead. See https://github.com/stackabletech/kafka-operator/pull/868[kafka-operator#868]. +===== Trino + +*Breaking:* +The PersistentVolumeClaims for Trino coordinator and workers have been removed because they caused problems due to storing its PID in `/stackable/data/var/run/launcher.pid`. +A forceful stop (eg. OOMKilled) could result in a leftover PID in this file. +In this case Trino would refuse startup with `ERROR: already running as 21`. +As the PersistentVolumeClaims didn't store any actual data, they have been removed. +See https://github.com/stackabletech/trino-operator/issues/768[trino-operator#768] and https://github.com/stackabletech/trino-operator/pull/769[trino-operator#769]. + +* Upgrading will result in an error, because Kubernetes currently does not allow changing the `volumeClaimTemplates` field. + Simply delete the mentioned StatefulSet and the operator will re-create it. +* Orphaned PVCs can be cleaned up. + Listing all Trino-related PVCs can be done using the following command: `kubectl get pvc -l app.kubernetes.io/name=trino`. +* The `.spec.coordinators|workers.config.resources.storage.data` field has been removed, as it is no longer needed. + ==== Supported versions ===== Product versions