diff --git a/.zuul.yaml b/.zuul.yaml index 9552fa3b47..e1dec5b3fd 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -38,6 +38,16 @@ nodes: - controller +- nodeset: + name: devstack-single-node-almalinux-10 + nodes: + - name: controller + label: almalinux-10-8GB + groups: + - name: tempest + nodes: + - controller + - nodeset: name: devstack-single-node-centos-9-stream nodes: @@ -49,38 +59,40 @@ - controller - nodeset: - name: devstack-single-node-opensuse-15 + name: devstack-single-node-centos-10-stream nodes: - name: controller - label: opensuse-15 + label: centos-10-stream-8GB groups: - name: tempest nodes: - controller - nodeset: - name: devstack-single-node-debian-bookworm + name: devstack-single-node-debian-trixie nodes: - name: controller - label: debian-bookworm + label: debian-trixie-8GB groups: - name: tempest nodes: - controller -# Note(sean-k-mooney): this is still used by horizon for -# horizon-integration-tests, horizon-integration-pytest and -# horizon-ui-pytest, remove when horizon is updated. - nodeset: - name: devstack-single-node-debian-bullseye + name: devstack-single-node-debian-bookworm nodes: - name: controller - label: debian-bullseye + label: debian-bookworm groups: - name: tempest nodes: - controller +# TODO(frickler): drop this dummy nodeset once all references have been removed +- nodeset: + name: devstack-single-node-opensuse-15 + nodes: [] + - nodeset: name: devstack-single-node-rockylinux-9 nodes: @@ -91,6 +103,46 @@ nodes: - controller +- nodeset: + name: devstack-single-node-rockylinux-10 + nodes: + - name: controller + label: rockylinux-10-8GB + groups: + - name: tempest + nodes: + - controller + +- nodeset: + name: openstack-two-node-centos-10-stream + nodes: + - name: controller + label: centos-10-stream-8GB + - name: compute1 + label: centos-10-stream-8GB + groups: + # Node where tests are executed and test results collected + - name: tempest + nodes: + - controller + # Nodes running the compute service + - name: compute + nodes: + - controller + - compute1 + # Nodes that are not the controller + - name: subnode + nodes: + - compute1 + # Switch node for multinode networking setup + - name: switch + nodes: + - controller + # Peer nodes for multinode networking setup + - name: peers + nodes: + - compute1 + - nodeset: name: openstack-two-node-centos-9-stream nodes: @@ -341,6 +393,36 @@ nodes: - compute1 +- nodeset: + name: devstack-two-node-debian-trixie + nodes: + - name: controller + label: debian-trixie-8GB + - name: compute1 + label: debian-trixie-8GB + groups: + # Node where tests are executed and test results collected + - name: tempest + nodes: + - controller + # Nodes running the compute service + - name: compute + nodes: + - controller + - compute1 + # Nodes that are not the controller + - name: subnode + nodes: + - compute1 + # Switch node for multinode networking setup + - name: switch + nodes: + - controller + # Peer nodes for multinode networking setup + - name: peers + nodes: + - compute1 + - job: name: devstack-base parent: openstack-multinode-fips @@ -357,6 +439,12 @@ nodes (everything but the controller). required-projects: - opendev.org/openstack/devstack + # this is a workaround for a packaging bug in ubuntu + # remove when https://bugs.launchpad.net/nova/+bug/2109592 + # is resolved and oslo.config is not a dep of the novnc deb + # via the defunct python3-novnc package. + - novnc/novnc + roles: - zuul: opendev.org/openstack/openstack-zuul-jobs vars: @@ -374,7 +462,6 @@ LOG_COLOR: false VERBOSE: true VERBOSE_NO_TIMESTAMP: true - NOVNC_FROM_PACKAGE: true ERROR_ON_CLONE: true # Gate jobs can't deal with nested virt. Disable it by default. LIBVIRT_TYPE: '{{ devstack_libvirt_type | default("qemu") }}' @@ -409,6 +496,7 @@ /etc/sudoers.d: logs '{{ stage_dir }}/iptables.txt': logs '{{ stage_dir }}/df.txt': logs + '{{ stage_dir }}/mount.txt': logs '{{ stage_dir }}/pip2-freeze.txt': logs '{{ stage_dir }}/pip3-freeze.txt': logs '{{ stage_dir }}/dpkg-l.txt': logs @@ -447,7 +535,6 @@ LOG_COLOR: false VERBOSE: true VERBOSE_NO_TIMESTAMP: true - NOVNC_FROM_PACKAGE: true ERROR_ON_CLONE: true LIBVIRT_TYPE: qemu devstack_services: @@ -620,7 +707,7 @@ ovsdb-server: true # Neutron services q-svc: true - q-ovn-metadata-agent: true + q-ovn-agent: true # Swift services s-account: true s-container: true @@ -657,7 +744,7 @@ ovs-vswitchd: true ovsdb-server: true # Neutron services - q-ovn-metadata-agent: true + q-ovn-agent: true # Cinder services c-bak: true c-vol: true @@ -730,6 +817,29 @@ # we often have to rush things through devstack to stabilise the gate, # and these platforms don't have the round-the-clock support to avoid # becoming blockers in that situation. +- job: + name: devstack-platform-almalinux-purple-lion-ovn-source + parent: tempest-full-py3 + description: AlmaLinux 10 platform test + nodeset: devstack-single-node-almalinux-10 + timeout: 9000 + voting: false + vars: + configure_swap_size: 4096 + devstack_localrc: + OVN_BUILD_FROM_SOURCE: True + OVN_BRANCH: "branch-24.03" + OVS_BRANCH: "branch-3.3" + OVS_SYSCONFDIR: "/usr/local/etc/openvswitch" + +- job: + name: devstack-platform-centos-10-stream + parent: tempest-full-py3 + description: CentOS 10 Stream platform test + nodeset: devstack-single-node-centos-10-stream + timeout: 9000 + voting: false + - job: name: devstack-platform-centos-9-stream parent: tempest-full-py3 @@ -742,6 +852,15 @@ timeout: 9000 voting: false +- job: + name: devstack-platform-debian-trixie + parent: tempest-full-py3 + description: Debian Trixie platform test + nodeset: devstack-single-node-debian-trixie + timeout: 9000 + vars: + configure_swap_size: 4096 + - job: name: devstack-platform-debian-bookworm parent: tempest-full-py3 @@ -769,13 +888,14 @@ PYTHON3_VERSION: 3.11 - job: - name: devstack-platform-ubuntu-jammy + name: devstack-platform-rocky-red-quartz parent: tempest-full-py3 - description: Ubuntu 22.04 LTS (Jammy) platform test - nodeset: openstack-single-node-jammy + description: Rocky Linux Red Quartz platform test + nodeset: devstack-single-node-rockylinux-10 timeout: 9000 + voting: false vars: - configure_swap_size: 8192 + configure_swap_size: 4096 - job: name: devstack-platform-ubuntu-noble-ovn-source @@ -912,16 +1032,19 @@ - devstack - devstack-ipv6 - devstack-enforce-scope + - devstack-platform-almalinux-purple-lion-ovn-source + - devstack-platform-centos-10-stream - devstack-platform-centos-9-stream - devstack-platform-debian-bookworm + - devstack-platform-debian-trixie - devstack-platform-rocky-blue-onyx + - devstack-platform-rocky-red-quartz - devstack-platform-ubuntu-noble-ovn-source - devstack-platform-ubuntu-noble-ovs - - devstack-platform-ubuntu-jammy - devstack-multinode - devstack-unit-tests - openstack-tox-bashate - - ironic-tempest-bios-ipmi-direct-tinyipa + - ironic-tempest-bios-ipmi-direct - swift-dsvm-functional - grenade: irrelevant-files: *common-irrelevant-files @@ -944,6 +1067,7 @@ - devstack - devstack-ipv6 - devstack-platform-debian-bookworm + - devstack-platform-debian-trixie - devstack-platform-ubuntu-noble # NOTE(danms): Disabled due to instability, see comment in the job # definition above. @@ -954,7 +1078,7 @@ - openstack-tox-bashate - neutron-ovs-grenade-multinode: irrelevant-files: *common-irrelevant-files - - ironic-tempest-bios-ipmi-direct-tinyipa + - ironic-tempest-bios-ipmi-direct - swift-dsvm-functional - grenade: irrelevant-files: *common-irrelevant-files @@ -995,9 +1119,11 @@ - devstack-no-tls-proxy periodic-weekly: jobs: + - devstack-platform-almalinux-purple-lion-ovn-source + - devstack-platform-centos-10-stream - devstack-platform-centos-9-stream - devstack-platform-debian-bookworm - devstack-platform-rocky-blue-onyx + - devstack-platform-rocky-red-quartz - devstack-platform-ubuntu-noble-ovn-source - devstack-platform-ubuntu-noble-ovs - - devstack-platform-ubuntu-jammy diff --git a/doc/source/index.rst b/doc/source/index.rst index 70871ef876..a07bb84922 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -40,7 +40,7 @@ Start with a clean and minimal install of a Linux system. DevStack attempts to support the two latest LTS releases of Ubuntu, Rocky Linux 9 and openEuler. -If you do not have a preference, Ubuntu 22.04 (Jammy) is the +If you do not have a preference, Ubuntu 24.04 (Noble) is the most tested, and will probably go the smoothest. Add Stack User (optional) diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst index 9185263443..560668f2a0 100644 --- a/doc/source/plugin-registry.rst +++ b/doc/source/plugin-registry.rst @@ -48,7 +48,6 @@ openstack/grian-ui `https://opendev.org/openstack/grian-ui openstack/heat `https://opendev.org/openstack/heat `__ openstack/heat-dashboard `https://opendev.org/openstack/heat-dashboard `__ openstack/ironic `https://opendev.org/openstack/ironic `__ -openstack/ironic-inspector `https://opendev.org/openstack/ironic-inspector `__ openstack/ironic-prometheus-exporter `https://opendev.org/openstack/ironic-prometheus-exporter `__ openstack/ironic-ui `https://opendev.org/openstack/ironic-ui `__ openstack/keystone `https://opendev.org/openstack/keystone `__ @@ -60,9 +59,6 @@ openstack/manila-tempest-plugin `https://opendev.org/openstack/manila-t openstack/manila-ui `https://opendev.org/openstack/manila-ui `__ openstack/masakari `https://opendev.org/openstack/masakari `__ openstack/mistral `https://opendev.org/openstack/mistral `__ -openstack/monasca-api `https://opendev.org/openstack/monasca-api `__ -openstack/monasca-events-api `https://opendev.org/openstack/monasca-events-api `__ -openstack/monasca-tempest-plugin `https://opendev.org/openstack/monasca-tempest-plugin `__ openstack/networking-bagpipe `https://opendev.org/openstack/networking-bagpipe `__ openstack/networking-baremetal `https://opendev.org/openstack/networking-baremetal `__ openstack/networking-bgpvpn `https://opendev.org/openstack/networking-bgpvpn `__ @@ -84,7 +80,6 @@ openstack/osprofiler `https://opendev.org/openstack/osprofil openstack/ovn-bgp-agent `https://opendev.org/openstack/ovn-bgp-agent `__ openstack/ovn-octavia-provider `https://opendev.org/openstack/ovn-octavia-provider `__ openstack/rally-openstack `https://opendev.org/openstack/rally-openstack `__ -openstack/shade `https://opendev.org/openstack/shade `__ openstack/skyline-apiserver `https://opendev.org/openstack/skyline-apiserver `__ openstack/storlets `https://opendev.org/openstack/storlets `__ openstack/tacker `https://opendev.org/openstack/tacker `__ @@ -114,6 +109,7 @@ starlingx/nfv `https://opendev.org/starlingx/nfv `__ vexxhost/openstack-operator `https://opendev.org/vexxhost/openstack-operator `__ x/almanach `https://opendev.org/x/almanach `__ +x/apmec `https://opendev.org/x/apmec `__ x/bilean `https://opendev.org/x/bilean `__ x/broadview-collector `https://opendev.org/x/broadview-collector `__ x/collectd-openstack-plugins `https://opendev.org/x/collectd-openstack-plugins `__ diff --git a/files/apache-cinder-api.template b/files/apache-cinder-api.template index e1246f11b6..e401803abc 100644 --- a/files/apache-cinder-api.template +++ b/files/apache-cinder-api.template @@ -6,21 +6,13 @@ Listen %PUBLICPORT% WSGIScriptAlias / %CINDER_BIN_DIR%/cinder-wsgi WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On - = 2.4> - ErrorLogFormat "%{cu}t %M" - + ErrorLogFormat "%{cu}t %M" ErrorLog /var/log/%APACHE_NAME%/c-api.log %SSLENGINE% %SSLCERTFILE% %SSLKEYFILE% - = 2.4> - Require all granted - - - Order allow,deny - Allow from all - + Require all granted diff --git a/files/apache-horizon.template b/files/apache-horizon.template index 98d02e168e..c6c55ecf27 100644 --- a/files/apache-horizon.template +++ b/files/apache-horizon.template @@ -10,7 +10,6 @@ DocumentRoot %HORIZON_DIR%/.blackhole/ Alias %WEBROOT%/media %HORIZON_DIR%/openstack_dashboard/static Alias %WEBROOT%/static %HORIZON_DIR%/static - Alias /static %HORIZON_DIR%/static RedirectMatch "^/$" "%WEBROOT%/" @@ -22,19 +21,9 @@ Options Indexes FollowSymLinks MultiViews AllowOverride None - # Apache 2.4 uses mod_authz_host for access control now (instead of - # "Allow") - - Order allow,deny - Allow from all - - = 2.4> - Require all granted - + Require all granted - = 2.4> - ErrorLogFormat "%{cu}t %M" - + ErrorLogFormat "%{cu}t %M" ErrorLog /var/log/%APACHE_NAME%/horizon_error.log LogLevel warn CustomLog /var/log/%APACHE_NAME%/horizon_access.log combined diff --git a/files/apache-nova-api.template b/files/apache-nova-api.template index bcf406edf3..66fcf73cf2 100644 --- a/files/apache-nova-api.template +++ b/files/apache-nova-api.template @@ -6,9 +6,7 @@ Listen %PUBLICPORT% WSGIScriptAlias / %PUBLICWSGI% WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On - = 2.4> - ErrorLogFormat "%M" - + ErrorLogFormat "%M" ErrorLog /var/log/%APACHE_NAME%/nova-api.log %SSLENGINE% %SSLCERTFILE% diff --git a/files/apache-nova-metadata.template b/files/apache-nova-metadata.template index 6231c1ced8..64be03166e 100644 --- a/files/apache-nova-metadata.template +++ b/files/apache-nova-metadata.template @@ -6,9 +6,7 @@ Listen %PUBLICPORT% WSGIScriptAlias / %PUBLICWSGI% WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On - = 2.4> - ErrorLogFormat "%M" - + ErrorLogFormat "%M" ErrorLog /var/log/%APACHE_NAME%/nova-metadata.log %SSLENGINE% %SSLCERTFILE% diff --git a/files/debs/general b/files/debs/general index 364f3cc6e2..0cddcf05f4 100644 --- a/files/debs/general +++ b/files/debs/general @@ -11,10 +11,8 @@ gettext # used for compiling message catalogs git graphviz # needed for docs iputils-ping -libapache2-mod-proxy-uwsgi libffi-dev # for pyOpenSSL libjpeg-dev # Pillow 3.0.0 -libpcre3-dev # for python-pcre libpq-dev # psycopg2 libssl-dev # for pyOpenSSL libsystemd-dev # for systemd-python @@ -35,5 +33,4 @@ tcpdump unzip uuid-runtime wget -wget zlib1g-dev diff --git a/files/rpms/general b/files/rpms/general index 8a5755cc37..6f4572c708 100644 --- a/files/rpms/general +++ b/files/rpms/general @@ -10,9 +10,10 @@ glibc-langpack-en # dist:rhel9 graphviz # needed only for docs httpd httpd-devel -iptables-nft # dist:rhel9 +iptables-nft # dist:rhel9,rhel10 iptables-services -java-1.8.0-openjdk-headless +java-1.8.0-openjdk-headless # not:rhel10 +java-21-openjdk-headless # dist:rhel10 libffi-devel libjpeg-turbo-devel # Pillow 3.0.0 libxml2-devel # lxml @@ -23,7 +24,8 @@ net-tools openssh-server openssl openssl-devel # to rebuild pyOpenSSL if needed -pcre-devel # for python-pcre +pcre2-devel # dist:rhel10 for python-pcre2 +pcre-devel # not:rhel10 for python-pcre pkgconfig postgresql-devel # psycopg2 psmisc diff --git a/files/rpms/n-cpu b/files/rpms/n-cpu index 7ce5a72d6b..3d50f3a062 100644 --- a/files/rpms/n-cpu +++ b/files/rpms/n-cpu @@ -1,10 +1,9 @@ cryptsetup dosfstools -genisoimage # not:rhel9 iscsi-initiator-utils libosinfo lvm2 sg3_utils # Stuff for diablo volumes sysfsutils -xorriso # not:rhel8 +xorriso diff --git a/files/rpms/nova b/files/rpms/nova index e0f13b854a..d0f843bb60 100644 --- a/files/rpms/nova +++ b/files/rpms/nova @@ -1,7 +1,6 @@ conntrack-tools curl ebtables -genisoimage # not:rhel9 required for config_drive iptables iputils kernel-modules # not:openEuler-22.03 @@ -11,4 +10,4 @@ polkit rabbitmq-server # NOPRIME sqlite sudo -xorriso # not:rhel8 +xorriso diff --git a/files/rpms/swift b/files/rpms/swift index 49a1833dc4..c3921a47d4 100644 --- a/files/rpms/swift +++ b/files/rpms/swift @@ -4,4 +4,3 @@ memcached rsync-daemon sqlite xfsprogs -xinetd # not:f36,rhel9 diff --git a/functions b/functions index 42d08d7c4a..63c6318c2e 100644 --- a/functions +++ b/functions @@ -47,6 +47,10 @@ function short_source { # export it so child shells have access to the 'short_source' function also. export -f short_source +EXTRA_FILES_RETRY=${EXTRA_FILES_RETRY:-3} +EXTRA_FILES_RETRY_ERRORS=${EXTRA_FILES_RETRY_ERRORS:-"500,503"} +EXTRA_FILES_DOWNLOAD_TIMEOUT=${EXTRA_FILES_DOWNLOAD_TIMEOUT:-2} +EXTRA_FILES_RETRY_TIMEOUT=${EXTRA_FILES_RETRY_TIMEOUT:-10} # Download a file from a URL # # Will check cache (in $FILES) or download given URL. @@ -55,17 +59,20 @@ export -f short_source # # Will echo the local path to the file as the output. Will die on # failure to download. -# + # Files can be pre-cached for CI environments, see EXTRA_CACHE_URLS # and tools/image_list.sh function get_extra_file { local file_url=$1 - - file_name=$(basename "$file_url") + local retry_args="--retry-on-host-error --retry-on-http-error=${EXTRA_FILES_RETRY_ERRORS} " + retry_args+="-t ${EXTRA_FILES_DOWNLOAD_TIMEOUT} --waitretry=${EXTRA_FILES_RETRY_TIMEOUT} " + retry_args+="--tries=${EXTRA_FILES_RETRY} --retry-connrefused" + # Using Bash parameter expansion (##*/) instead of external 'basename' + local file_name="${file_url##*/}" if [[ $file_url != file* ]]; then # If the file isn't cache, download it if [[ ! -f $FILES/$file_name ]]; then - wget --progress=dot:giga -t 2 -c $file_url -O $FILES/$file_name + wget --progress=dot:giga ${retry_args} -c $file_url -O $FILES/$file_name if [[ $? -ne 0 ]]; then die "$file_url could not be downloaded" fi @@ -74,7 +81,7 @@ function get_extra_file { return else # just strip the file:// bit and that's the path to the file - echo $file_url | sed 's/$file:\/\///g' + echo "${file_url#file://}" fi } @@ -147,7 +154,8 @@ function upload_image { if [[ $rc -ne 0 ]]; then if [[ "$attempt" -eq "$max_attempts" ]]; then echo "Not found: $image_url" - return + # Signal failure to download to the caller, so they can fail early + return 1 fi echo "Download failed, retrying in $attempt second, attempt: $attempt" sleep $attempt diff --git a/functions-common b/functions-common index e265256ccf..39424e3352 100644 --- a/functions-common +++ b/functions-common @@ -43,6 +43,9 @@ declare -A -g GITREPO declare -A -g GITBRANCH declare -A -g GITDIR +# Systemd service file environment variables per service +declare -A -g SYSTEMD_ENV_VARS + KILL_PATH="$(which kill)" # Save these variables to .stackenv @@ -420,7 +423,7 @@ function _ensure_lsb_release { elif [[ -x $(command -v zypper 2>/dev/null) ]]; then sudo zypper -n install lsb-release elif [[ -x $(command -v dnf 2>/dev/null) ]]; then - sudo dnf install -y redhat-lsb-core || sudo dnf install -y openeuler-lsb + sudo dnf install -y python3-distro || sudo dnf install -y openeuler-lsb else die $LINENO "Unable to find or auto-install lsb_release" fi @@ -433,15 +436,12 @@ function _ensure_lsb_release { # - os_VENDOR # - os_PACKAGE function GetOSVersion { - # CentOS Stream 9 and RHEL 9 do not provide lsb_release + # CentOS Stream 9 or later and RHEL 9 or later do not provide lsb_release source /etc/os-release - if [[ "${ID}${VERSION}" == "centos9" ]] || [[ "${ID}${VERSION}" =~ "rhel9" ]]; then + if [[ "${ID}" =~ (almalinux|centos|rocky|rhel) ]]; then os_RELEASE=${VERSION_ID} - os_CODENAME="n/a" + os_CODENAME=$(echo $VERSION | grep -oP '(?<=[(])[^)]*' || echo 'n/a') os_VENDOR=$(echo $NAME | tr -d '[:space:]') - elif [[ "${ID}${VERSION}" =~ "rocky9" ]]; then - os_VENDOR="Rocky" - os_RELEASE=${VERSION_ID} else _ensure_lsb_release @@ -480,11 +480,10 @@ function GetDistro { "$os_VENDOR" =~ (AlmaLinux) || \ "$os_VENDOR" =~ (Scientific) || \ "$os_VENDOR" =~ (OracleServer) || \ - "$os_VENDOR" =~ (Rocky) || \ + "$os_VENDOR" =~ (RockyLinux) || \ "$os_VENDOR" =~ (Virtuozzo) ]]; then - # Drop the . release as we assume it's compatible - # XXX re-evaluate when we get RHEL10 - DISTRO="rhel${os_RELEASE::1}" + MAJOR_VERSION=$(echo $os_RELEASE | cut -d. -f1) + DISTRO="rhel${MAJOR_VERSION}" elif [[ "$os_VENDOR" =~ (openEuler) ]]; then DISTRO="openEuler-$os_RELEASE" else @@ -518,17 +517,6 @@ function is_arch { [[ "$(uname -m)" == "$1" ]] } -# Determine if current distribution is an Oracle distribution -# is_oraclelinux -function is_oraclelinux { - if [[ -z "$os_VENDOR" ]]; then - GetOSVersion - fi - - [ "$os_VENDOR" = "OracleServer" ] -} - - # Determine if current distribution is a Fedora-based distribution # (Fedora, RHEL, CentOS, Rocky, etc). # is_fedora @@ -542,7 +530,7 @@ function is_fedora { [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \ [ "$os_VENDOR" = "RedHatEnterprise" ] || \ [ "$os_VENDOR" = "RedHatEnterpriseLinux" ] || \ - [ "$os_VENDOR" = "Rocky" ] || \ + [ "$os_VENDOR" = "RockyLinux" ] || \ [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "CentOSStream" ] || \ [ "$os_VENDOR" = "AlmaLinux" ] || \ [ "$os_VENDOR" = "OracleServer" ] || [ "$os_VENDOR" = "Virtuozzo" ] @@ -1642,6 +1630,9 @@ function _run_under_systemd { user=$STACK_USER fi local env_vars="$5" + if [[ -v SYSTEMD_ENV_VARS[$service] ]]; then + env_vars="${SYSTEMD_ENV_VARS[$service]} $env_vars" + fi if [[ "$command" =~ "uwsgi" ]] ; then if [[ "$GLOBAL_VENV" == "True" ]] ; then cmd="$cmd --venv $DEVSTACK_VENV" diff --git a/inc/ini-config b/inc/ini-config index f65e42d3a5..920d4775fa 100644 --- a/inc/ini-config +++ b/inc/ini-config @@ -189,8 +189,10 @@ function iniset { local option=$3 local value=$4 - # Escape the ampersand character (&) - value=$(echo $value | sed -e 's/&/\\&/g') + # Escape the ampersand (&) and backslash (\) characters for sed + # Order of substitution matters: we escape backslashes first before + # adding more backslashes to escape ampersands + value=$(echo $value | sed -e 's/\\/\\\\/g' -e 's/&/\\&/g') if [[ -z $section || -z $option ]]; then $xtrace diff --git a/inc/meta-config b/inc/meta-config index b9d9649e4b..1215bb8307 100644 --- a/inc/meta-config +++ b/inc/meta-config @@ -90,7 +90,6 @@ function merge_config_file { local real_configfile real_configfile=$(eval echo $configfile) if [ ! -f $real_configfile ]; then - mkdir -p $(dirname $real_configfile) || die $LINENO "could not create the directory of $real_configfile ($configfile)" touch $real_configfile || die $LINENO "could not create config file $real_configfile ($configfile)" fi @@ -186,11 +185,15 @@ function merge_config_group { break fi dir=$(dirname $realconfigfile) - if [[ -d $dir ]]; then - merge_config_file $localfile $group $configfile - else - die $LINENO "bogus config file specification $configfile ($configfile=$realconfigfile, $dir is not a directory)" + + test -e $dir && ! test -d $dir && die $LINENO "bogus config file specification $configfile ($configfile=$realconfigfile, $dir exists but it is not a directory)" + + if ! [[ -e $dir ]] ; then + sudo mkdir -p $dir || die $LINENO "could not create the directory of $real_configfile ($configfile)" + sudo chown ${STACK_USER} $dir fi + + merge_config_file $localfile $group $configfile done done } diff --git a/inc/python b/inc/python index cd90ac82c6..3969c1fa82 100644 --- a/inc/python +++ b/inc/python @@ -273,8 +273,7 @@ function use_library_from_git { function lib_installed_from_git { local name=$1 local safe_name - safe_name=$(python -c "from pkg_resources import safe_name; \ - print(safe_name('${name}'))") + safe_name=$(python -c "from packaging import canonicalize_name; print(canonicalize_name('${name}'))") # Note "pip freeze" doesn't always work here, because it tries to # be smart about finding the remote of the git repo the package # was installed from. This doesn't work with zuul which clones diff --git a/lib/apache b/lib/apache index 449d2e70d4..b3379a7cde 100644 --- a/lib/apache +++ b/lib/apache @@ -82,14 +82,14 @@ function install_apache_uwsgi { fi if is_ubuntu; then - local pkg_list="uwsgi uwsgi-plugin-python3 libapache2-mod-proxy-uwsgi" + local pkg_list="uwsgi uwsgi-plugin-python3" install_package ${pkg_list} # NOTE(ianw) 2022-02-03 : Fedora 35 needs to skip this and fall # into the install-from-source because the upstream packages # didn't fix Python 3.10 compatibility before release. Should be # fixed in uwsgi 4.9.0; can remove this when packages available # or we drop this release - elif is_fedora && ! is_openeuler && ! [[ $DISTRO =~ f36|rhel9 ]]; then + elif is_fedora && ! is_openeuler && ! [[ $DISTRO =~ rhel9 ]]; then # Note httpd comes with mod_proxy_uwsgi and it is loaded by # default; the mod_proxy_uwsgi package actually conflicts now. # See: diff --git a/lib/cinder b/lib/cinder index b557d4b10b..80ffcd0df4 100644 --- a/lib/cinder +++ b/lib/cinder @@ -59,6 +59,7 @@ else fi CINDER_STATE_PATH=${CINDER_STATE_PATH:=$DATA_DIR/cinder} +OS_BRICK_LOCK_PATH=${OS_BRICK_LOCK_PATH:=$DATA_DIR/os_brick} CINDER_CONF_DIR=/etc/cinder CINDER_CONF=$CINDER_CONF_DIR/cinder.conf @@ -303,15 +304,6 @@ function configure_cinder { cp $CINDER_DIR/etc/cinder/api-paste.ini $CINDER_API_PASTE_INI - inicomment $CINDER_API_PASTE_INI filter:authtoken auth_host - inicomment $CINDER_API_PASTE_INI filter:authtoken auth_port - inicomment $CINDER_API_PASTE_INI filter:authtoken auth_protocol - inicomment $CINDER_API_PASTE_INI filter:authtoken cafile - inicomment $CINDER_API_PASTE_INI filter:authtoken admin_tenant_name - inicomment $CINDER_API_PASTE_INI filter:authtoken admin_user - inicomment $CINDER_API_PASTE_INI filter:authtoken admin_password - inicomment $CINDER_API_PASTE_INI filter:authtoken signing_dir - configure_keystone_authtoken_middleware $CINDER_CONF cinder iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL @@ -419,8 +411,11 @@ function configure_cinder { iniset $CINDER_CONF DEFAULT glance_ca_certificates_file $SSL_BUNDLE_FILE fi + # Set glance credentials (used for location APIs) + configure_keystone_authtoken_middleware $CINDER_CONF glance glance + # Set nova credentials (used for os-assisted-snapshots) - configure_keystone_authtoken_middleware $CINDER_CONF nova nova + configure_keystoneauth $CINDER_CONF nova nova iniset $CINDER_CONF nova region_name "$REGION_NAME" iniset $CINDER_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT" @@ -511,6 +506,21 @@ function init_cinder { mkdir -p $CINDER_STATE_PATH/volumes } + +function init_os_brick { + mkdir -p $OS_BRICK_LOCK_PATH + if is_service_enabled cinder; then + iniset $CINDER_CONF os_brick lock_path $OS_BRICK_LOCK_PATH + fi + if is_service_enabled nova; then + iniset $NOVA_CONF os_brick lock_path $OS_BRICK_LOCK_PATH + fi + if is_service_enabled glance; then + iniset $GLANCE_API_CONF os_brick lock_path $OS_BRICK_LOCK_PATH + iniset $GLANCE_CACHE_CONF os_brick lock_path $OS_BRICK_LOCK_PATH + fi +} + # install_cinder() - Collect source and prepare function install_cinder { git_clone $CINDER_REPO $CINDER_DIR $CINDER_BRANCH @@ -730,9 +740,8 @@ function configure_cinder_volume_upload { } function init_cinder_service_user_conf { - configure_keystone_authtoken_middleware $CINDER_CONF cinder service_user iniset $CINDER_CONF service_user send_service_user_token True - iniset $CINDER_CONF service_user auth_strategy keystone + configure_keystoneauth $CINDER_CONF cinder service_user } # Restore xtrace diff --git a/lib/cinder_backups/ceph b/lib/cinder_backups/ceph index 4b180490d7..e4d6b96407 100644 --- a/lib/cinder_backups/ceph +++ b/lib/cinder_backups/ceph @@ -19,6 +19,7 @@ set +o xtrace # Defaults # -------- +CINDER_BAK_CEPH_MAX_SNAPSHOTS=${CINDER_BAK_CEPH_MAX_SNAPSHOTS:-0} CINDER_BAK_CEPH_POOL=${CINDER_BAK_CEPH_POOL:-backups} CINDER_BAK_CEPH_POOL_PG=${CINDER_BAK_CEPH_POOL_PG:-8} CINDER_BAK_CEPH_POOL_PGP=${CINDER_BAK_CEPH_POOL_PGP:-8} @@ -32,12 +33,13 @@ function configure_cinder_backup_ceph { if [[ "$REMOTE_CEPH" = "False" && "$CEPH_REPLICAS" -ne 1 ]]; then sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID} fi - sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring - sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring + sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "profile rbd" osd "profile rbd pool=${CINDER_BAK_CEPH_POOL}, profile rbd pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring + sudo chown $STACK_USER ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring fi iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.ceph.CephBackupDriver" iniset $CINDER_CONF DEFAULT backup_ceph_conf "$CEPH_CONF_FILE" + iniset $CINDER_CONF DEFAULT backup_ceph_max_snapshots "$CINDER_BAK_CEPH_MAX_SNAPSHOTS" iniset $CINDER_CONF DEFAULT backup_ceph_pool "$CINDER_BAK_CEPH_POOL" iniset $CINDER_CONF DEFAULT backup_ceph_user "$CINDER_BAK_CEPH_USER" iniset $CINDER_CONF DEFAULT backup_ceph_stripe_unit 0 diff --git a/lib/databases/mysql b/lib/databases/mysql index 629014c1d8..4def1842a7 100644 --- a/lib/databases/mysql +++ b/lib/databases/mysql @@ -18,9 +18,9 @@ register_database mysql if [[ -z "$MYSQL_SERVICE_NAME" ]]; then MYSQL_SERVICE_NAME=mysql - if is_fedora && ! is_oraclelinux; then + if is_fedora; then MYSQL_SERVICE_NAME=mariadb - elif [[ "$DISTRO" =~ bookworm|bullseye ]]; then + elif [[ "$DISTRO" =~ trixie|bookworm|bullseye ]]; then MYSQL_SERVICE_NAME=mariadb fi fi @@ -44,15 +44,9 @@ function cleanup_database_mysql { apt_get purge -y mysql* mariadb* sudo rm -rf /var/lib/mysql sudo rm -rf /etc/mysql - return - elif is_oraclelinux; then - uninstall_package mysql-community-server - sudo rm -rf /var/lib/mysql elif is_fedora; then uninstall_package mariadb-server sudo rm -rf /var/lib/mysql - else - return fi } @@ -68,8 +62,6 @@ function configure_database_mysql { if is_ubuntu; then my_conf=/etc/mysql/my.cnf - elif is_oraclelinux; then - my_conf=/etc/my.cnf elif is_fedora; then my_conf=/etc/my.cnf local cracklib_conf=/etc/my.cnf.d/cracklib_password_check.cnf @@ -101,13 +93,20 @@ function configure_database_mysql { restart_service $MYSQL_SERVICE_NAME fi + # MariaDB 10.4+ on modern Debian/Ubuntu uses unix_socket auth by default + # See https://mariadb.org/authentication-in-mariadb-10-4/ + local use_mariadb_socket_auth=False + if is_ubuntu && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then + if [[ ! "$DISTRO" =~ bookworm|bullseye ]]; then + use_mariadb_socket_auth=True + fi + fi + # Set the root password - only works the first time. For Ubuntu, we already # did that with debconf before installing the package, but we still try, # because the package might have been installed already. We don't do this - # for Ubuntu 22.04+ because the authorization model change in - # version 10.4 of mariadb. See - # https://mariadb.org/authentication-in-mariadb-10-4/ - if ! (is_ubuntu && [[ ! "$DISTRO" =~ bookworm|bullseye ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]); then + # for MariaDB with socket auth because the root password is managed differently. + if [[ "$use_mariadb_socket_auth" != "True" ]]; then sudo mysqladmin -u root password $DATABASE_PASSWORD || true fi @@ -119,19 +118,30 @@ function configure_database_mysql { local cmd_args="-uroot -p$DATABASE_PASSWORD -h$SERVICE_LOCAL_HOST " fi - # In mariadb e.g. on Ubuntu socket plugin is used for authentication - # as root so it works only as sudo. To restore old "mysql like" behaviour, - # we need to change auth plugin for root user - # TODO(frickler): simplify this logic - if is_ubuntu && [[ ! "$DISTRO" =~ bookworm|bullseye ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then - # For Ubuntu 22.04+ we follow the model outlined in - # https://mariadb.org/authentication-in-mariadb-10-4/ - sudo mysql -e "ALTER USER $DATABASE_USER@localhost IDENTIFIED VIA mysql_native_password USING PASSWORD('$DATABASE_PASSWORD');" + # Workaround for mariadb > 11.6.2, + # see https://bugs.launchpad.net/nova/+bug/2116186/comments/3 + min_db_ver="11.6.2" + db_version=$(sudo mysql ${cmd_args} -e "select version();" -sN | cut -d '-' -f 1) + max_db_ver=$(printf '%s\n' ${min_db_ver} ${db_version} | sort -V | tail -n 1) + if [[ "${min_db_ver}" != "${max_db_ver}" ]]; then + iniset -sudo $my_conf mysqld innodb_snapshot_isolation OFF + restart_service $MYSQL_SERVICE_NAME + fi + + # Configure database user authentication + if [[ "$use_mariadb_socket_auth" == "True" ]]; then + # Allow both unix_socket (for sudo mysql) and password auth + # Using OR allows restacking without needing to reset auth in unstack + sudo mysql -e "ALTER USER $DATABASE_USER@localhost IDENTIFIED VIA unix_socket OR mysql_native_password USING PASSWORD('$DATABASE_PASSWORD');" fi - if ! (is_ubuntu && [[ ! "$DISTRO" =~ bookworm|bullseye ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]); then - # Create DB user if it does not already exist + + # Create remote access user and grant privileges (needed for all distros) + if [[ "$use_mariadb_socket_auth" == "True" ]]; then + # Use sudo mysql since we have socket auth + sudo mysql -e "CREATE USER IF NOT EXISTS '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';" + sudo mysql -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%';" + else sudo mysql $cmd_args -e "CREATE USER IF NOT EXISTS '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';" - # Update the DB to give user '$DATABASE_USER'@'%' full control of the all databases: sudo mysql $cmd_args -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%';" fi @@ -216,9 +226,7 @@ EOF fi # Install mysql-server if [[ "$INSTALL_DATABASE_SERVER_PACKAGES" == "True" ]]; then - if is_oraclelinux; then - install_package mysql-community-server - elif is_fedora; then + if is_fedora; then install_package mariadb-server mariadb-devel mariadb sudo systemctl enable $MYSQL_SERVICE_NAME elif is_ubuntu; then diff --git a/lib/glance b/lib/glance index b596b53271..9422c22141 100644 --- a/lib/glance +++ b/lib/glance @@ -83,13 +83,6 @@ GLANCE_DEFAULT_BACKEND=${GLANCE_DEFAULT_BACKEND:-fast} GLANCE_CACHE_DIR=${GLANCE_CACHE_DIR:=$DATA_DIR/glance/cache} GLANCE_CACHE_DRIVER=${GLANCE_CACHE_DRIVER:-centralized_db} -# Full Glance functionality requires running in standalone mode. If we are -# not in uwsgi mode, then we are standalone, otherwise allow separate control. -if [[ "$WSGI_MODE" != "uwsgi" ]]; then - GLANCE_STANDALONE=True -fi -GLANCE_STANDALONE=${GLANCE_STANDALONE:-False} - # File path for each store specified in GLANCE_MULTIPLE_FILE_STORES, the store # identifier will be appended to this path at runtime. If GLANCE_MULTIPLE_FILE_STORES # has fast,cheap specified then filepath will be generated like $DATA_DIR/glance/fast @@ -139,14 +132,7 @@ GLANCE_UWSGI_CONF=$GLANCE_CONF_DIR/glance-uwsgi.ini # Glance default limit for Devstack GLANCE_LIMIT_IMAGE_SIZE_TOTAL=${GLANCE_LIMIT_IMAGE_SIZE_TOTAL:-2000} -# If wsgi mode is uwsgi run glance under uwsgi, else default to eventlet -# TODO(mtreinish): Remove the eventlet path here and in all the similar -# conditionals below after the Pike release -if [[ "$WSGI_MODE" == "uwsgi" ]]; then - GLANCE_URL="$GLANCE_SERVICE_PROTOCOL://$GLANCE_SERVICE_HOST/image" -else - GLANCE_URL="$GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT" -fi +GLANCE_URL="$GLANCE_SERVICE_PROTOCOL://$GLANCE_SERVICE_HOST/image" # Functions # --------- @@ -451,12 +437,11 @@ function configure_glance { iniset $GLANCE_CACHE_CONF glance_store filesystem_store_datadir $GLANCE_IMAGE_DIR/ # Set default configuration options for the glance-image-import - iniset $GLANCE_IMAGE_IMPORT_CONF image_import_opts image_import_plugins [] + iniset $GLANCE_IMAGE_IMPORT_CONF image_import_opts image_import_plugins "[]" iniset $GLANCE_IMAGE_IMPORT_CONF inject_metadata_properties ignore_user_roles admin iniset $GLANCE_IMAGE_IMPORT_CONF inject_metadata_properties inject cp -p $GLANCE_DIR/etc/schema-image.json $GLANCE_SCHEMA_JSON - cp -p $GLANCE_DIR/etc/metadefs/*.json $GLANCE_METADEF_DIR if is_service_enabled tls-proxy; then @@ -467,19 +452,15 @@ function configure_glance { iniset $GLANCE_CACHE_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/%(project_id)s" fi - if [[ "$GLANCE_STANDALONE" == False ]]; then - write_local_uwsgi_http_config "$GLANCE_UWSGI_CONF" "$GLANCE_UWSGI" "/image" "glance-api" - # Grab our uwsgi listen address and use that to fill out our - # worker_self_reference_url config - iniset $GLANCE_API_CONF DEFAULT worker_self_reference_url \ - $(awk '-F= ' '/^http-socket/ { print "http://"$2}' $GLANCE_UWSGI_CONF) - else - write_local_proxy_http_config glance "http://$GLANCE_SERVICE_HOST:$GLANCE_SERVICE_PORT_INT" "/image" - iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS - iniset $GLANCE_API_CONF DEFAULT bind_port $GLANCE_SERVICE_PORT_INT - iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS" - iniset $GLANCE_API_CONF DEFAULT worker_self_reference_url $GLANCE_URL - fi + write_local_uwsgi_http_config "$GLANCE_UWSGI_CONF" "$GLANCE_UWSGI" "/image" "glance-api" + + # Grab our uwsgi listen address and use that to fill out our + # worker_self_reference_url config + iniset $GLANCE_API_CONF DEFAULT worker_self_reference_url $(awk '-F= ' '/^http-socket/ { print "http://"$2}' $GLANCE_UWSGI_CONF) + + # Configure the Python binary used for "import" plugins. If unset, these + # will attempt the uwsgi binary instead. + iniset $GLANCE_API_CONF wsgi python_interpreter $PYTHON if [[ "$GLANCE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then iniset $GLANCE_API_CONF oslo_policy enforce_scope true @@ -503,7 +484,9 @@ function configure_glance { function create_glance_accounts { if is_service_enabled g-api; then - create_service_user "glance" + # When cinder talk to glance service APIs user needs service + # role for RBAC checks and admin role for cinder to access images. + create_service_user "glance" "admin" # required for swift access if is_service_enabled s-proxy; then @@ -664,17 +647,8 @@ function start_glance_remote_clone { # start_glance() - Start running processes function start_glance { local service_protocol=$GLANCE_SERVICE_PROTOCOL - if is_service_enabled tls-proxy; then - if [[ "$WSGI_MODE" != "uwsgi" ]]; then - start_tls_proxy glance-service '*' $GLANCE_SERVICE_PORT $GLANCE_SERVICE_HOST $GLANCE_SERVICE_PORT_INT - fi - fi - if [[ "$GLANCE_STANDALONE" == False ]]; then - run_process g-api "$(which uwsgi) --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" - else - run_process g-api "$GLANCE_BIN_DIR/glance-api --config-dir=$GLANCE_CONF_DIR" - fi + run_process g-api "$(which uwsgi) --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" if is_service_enabled g-api-r; then echo "Starting the g-api-r clone service..." diff --git a/lib/keystone b/lib/keystone index 8371045026..2077916d80 100644 --- a/lib/keystone +++ b/lib/keystone @@ -195,6 +195,10 @@ function configure_keystone { iniset $KEYSTONE_CONF cache backend $CACHE_BACKEND iniset $KEYSTONE_CONF cache memcache_servers $MEMCACHE_SERVERS + # Enable errors if response validation fails. We want this enabled in CI + # and development contexts to highlights bugs in our response schemas. + iniset $KEYSTONE_CONF api response_validation error + iniset_rpc_backend keystone $KEYSTONE_CONF oslo_messaging_notifications local service_port=$KEYSTONE_SERVICE_PORT @@ -421,17 +425,13 @@ function create_service_user { fi } -# Configure a service to use the auth token middleware. -# -# configure_keystone_authtoken_middleware conf_file admin_user IGNORED [section] +# Configure options for keystoneauth # -# section defaults to keystone_authtoken, which is where auth_token looks in -# the .conf file. If the paste config file is used (api-paste.ini) then -# provide the section name for the auth_token filter. -function configure_keystone_authtoken_middleware { +# configure_keystoneauth conf_file admin_user section +function configure_keystoneauth { local conf_file=$1 local admin_user=$2 - local section=${3:-keystone_authtoken} + local section=$3 iniset $conf_file $section auth_type password iniset $conf_file $section interface public @@ -441,9 +441,28 @@ function configure_keystone_authtoken_middleware { iniset $conf_file $section user_domain_name "$SERVICE_DOMAIN_NAME" iniset $conf_file $section project_name $SERVICE_PROJECT_NAME iniset $conf_file $section project_domain_name "$SERVICE_DOMAIN_NAME" +} +# Configure a service to use the auth token middleware. +# +# configure_keystone_authtoken_middleware conf_file admin_user [section] +# +# section defaults to keystone_authtoken, which is where auth_token looks in +# the .conf file. If the paste config file is used (api-paste.ini) then +# provide the section name for the auth_token filter. +function configure_keystone_authtoken_middleware { + local conf_file=$1 + local admin_user=$2 + local section=${3:-keystone_authtoken} + local service_type=$4 + + configure_keystoneauth $conf_file $admin_user $section + iniset $conf_file $section www_authenticate_uri $KEYSTONE_SERVICE_URI iniset $conf_file $section cafile $SSL_BUNDLE_FILE iniset $conf_file $section memcached_servers $MEMCACHE_SERVERS + if [[ -n "$service_type" ]]; then + iniset $conf_file $section service_type $service_type + fi } # configure_auth_token_middleware conf_file admin_user IGNORED [section] @@ -608,6 +627,7 @@ function create_ldap_domain { iniset $KEYSTONE_LDAP_DOMAIN_FILE ldap user_name_attribute "cn" iniset $KEYSTONE_LDAP_DOMAIN_FILE ldap user_mail_attribute "mail" iniset $KEYSTONE_LDAP_DOMAIN_FILE ldap user_id_attribute "uid" + iniset $KEYSTONE_LDAP_DOMAIN_FILE ldap user_enabled_emulation "True" iniset $KEYSTONE_LDAP_DOMAIN_FILE ldap user "cn=Manager,dc=openstack,dc=org" iniset $KEYSTONE_LDAP_DOMAIN_FILE ldap url "ldap://localhost" iniset $KEYSTONE_LDAP_DOMAIN_FILE ldap suffix $LDAP_BASE_DN diff --git a/lib/ldap b/lib/ldap index b0195db258..66c2afc4d5 100644 --- a/lib/ldap +++ b/lib/ldap @@ -82,6 +82,14 @@ function init_ldap { # Remove data but not schemas clear_ldap_state + if is_ubuntu; then + # a bug in OpenLDAP 2.6.7+ + # (https://bugs.openldap.org/show_bug.cgi?id=10336) causes slapd crash + # after deleting nonexisting tree. It is fixed upstream, but Ubuntu is + # still not having a fix in Noble. Try temporarily simly restarting the + # process. + sudo service $LDAP_SERVICE_NAME restart + fi # Add our top level ldap nodes if ldapsearch -x -w $LDAP_PASSWORD -D "$LDAP_MANAGER_DN" -H $LDAP_URL -b "$LDAP_BASE_DN" | grep -q "Success"; then diff --git a/lib/libraries b/lib/libraries index 26254c75af..ffc004b24c 100755 --- a/lib/libraries +++ b/lib/libraries @@ -27,6 +27,7 @@ GITDIR["castellan"]=$DEST/castellan GITDIR["cliff"]=$DEST/cliff GITDIR["cursive"]=$DEST/cursive GITDIR["debtcollector"]=$DEST/debtcollector +GITDIR["etcd3gw"]=$DEST/etcd3gw GITDIR["futurist"]=$DEST/futurist GITDIR["openstacksdk"]=$DEST/openstacksdk GITDIR["os-client-config"]=$DEST/os-client-config @@ -131,8 +132,12 @@ function install_libs { # python client libraries we might need from git can go here _install_lib_from_source "python-barbicanclient" - # etcd (because tooz does not have a hard dependency on these) - pip_install etcd3gw + if use_library_from_git etcd3gw ; then + _install_lib_from_source "etcd3gw" + else + # etcd (because tooz does not have a hard dependency on these) + pip_install etcd3gw + fi } # Restore xtrace diff --git a/lib/neutron b/lib/neutron index ea2d8e728a..f28ec9a6fe 100644 --- a/lib/neutron +++ b/lib/neutron @@ -201,35 +201,35 @@ fi # -------------------------------- # The following variables control the Neutron ML2 plugins' allocation -# of tenant networks and availability of provider networks. If these -# are not configured in ``localrc``, tenant networks will be local to +# of project networks and availability of provider networks. If these +# are not configured in ``localrc``, project networks will be local to # the host (with no remote connectivity), and no physical resources # will be available for the allocation of provider networks. -# To disable tunnels (GRE or VXLAN) for tenant networks, +# To disable tunnels (GRE or VXLAN) for project networks, # set to False in ``local.conf``. # GRE tunnels are only supported by the openvswitch. ENABLE_TENANT_TUNNELS=${ENABLE_TENANT_TUNNELS:-True} -# If using GRE, VXLAN or GENEVE tunnels for tenant networks, -# specify the range of IDs from which tenant networks are +# If using GRE, VXLAN or GENEVE tunnels for project networks, +# specify the range of IDs from which project networks are # allocated. Can be overridden in ``localrc`` if necessary. TENANT_TUNNEL_RANGES=${TENANT_TUNNEL_RANGES:-1:1000} -# To use VLANs for tenant networks, set to True in localrc. VLANs +# To use VLANs for project networks, set to True in localrc. VLANs # are supported by the ML2 plugins, requiring additional configuration # described below. ENABLE_TENANT_VLANS=${ENABLE_TENANT_VLANS:-False} -# If using VLANs for tenant networks, set in ``localrc`` to specify -# the range of VLAN VIDs from which tenant networks are +# If using VLANs for project networks, set in ``localrc`` to specify +# the range of VLAN VIDs from which project networks are # allocated. An external network switch must be configured to # trunk these VLANs between hosts for multi-host connectivity. # # Example: ``TENANT_VLAN_RANGE=1000:1999`` TENANT_VLAN_RANGE=${TENANT_VLAN_RANGE:-} -# If using VLANs for tenant networks, or if using flat or VLAN +# If using VLANs for project networks, or if using flat or VLAN # provider networks, set in ``localrc`` to the name of the physical # network, and also configure ``OVS_PHYSICAL_BRIDGE`` for the # openvswitch agent, as described below. @@ -237,7 +237,7 @@ TENANT_VLAN_RANGE=${TENANT_VLAN_RANGE:-} # Example: ``PHYSICAL_NETWORK=default`` PHYSICAL_NETWORK=${PHYSICAL_NETWORK:-public} -# With the openvswitch agent, if using VLANs for tenant networks, +# With the openvswitch agent, if using VLANs for project networks, # or if using flat or VLAN provider networks, set in ``localrc`` to # the name of the OVS bridge to use for the physical network. The # bridge will be created if it does not already exist, but a @@ -348,12 +348,6 @@ function _determine_config_l3 { echo "$opts" } -function _enable_ovn_maintenance { - if [[ $Q_AGENT == "ovn" ]]; then - enable_service neutron-ovn-maintenance-worker - fi -} - function _run_ovn_maintenance { if [[ $Q_AGENT == "ovn" ]]; then run_process neutron-ovn-maintenance-worker "$NEUTRON_BIN_DIR/neutron-ovn-maintenance-worker $cfg_file_options" @@ -460,14 +454,7 @@ function configure_neutron_nova { function create_nova_conf_neutron { local conf=${1:-$NOVA_CONF} - iniset $conf neutron auth_type "password" - iniset $conf neutron auth_url "$KEYSTONE_SERVICE_URI" - iniset $conf neutron username nova - iniset $conf neutron password "$SERVICE_PASSWORD" - iniset $conf neutron user_domain_name "$SERVICE_DOMAIN_NAME" - iniset $conf neutron project_name "$SERVICE_PROJECT_NAME" - iniset $conf neutron project_domain_name "$SERVICE_DOMAIN_NAME" - iniset $conf neutron auth_strategy "$Q_AUTH_STRATEGY" + configure_keystoneauth $conf nova neutron iniset $conf neutron region_name "$REGION_NAME" # optionally set options in nova_conf @@ -606,6 +593,39 @@ function start_ovn_services { fi } +# Enable neutron server services based on configuration +# This function determines which neutron services should be enabled +# and adds them to ENABLED_SERVICES. It reads the neutron configuration +# to determine if RPC workers should be enabled. +# This must be called after configure_neutron has created the config files. +function enable_neutron_server_services { + local rpc_workers + + # The default value of "rpc_workers" is None (not defined). If + # "rpc_workers" is explicitly set to 0, the RPC workers process + # should not be executed. + if [[ -f $NEUTRON_CONF ]]; then + rpc_workers=$(iniget_multiline $NEUTRON_CONF DEFAULT rpc_workers) + else + # If config doesn't exist yet, assume default behavior (enable rpc workers) + rpc_workers="" + fi + + # Always enable these core services + enable_service neutron-api + enable_service neutron-periodic-workers + + # Conditionally enable RPC server based on configuration + if [[ "$rpc_workers" != "0" ]]; then + enable_service neutron-rpc-server + fi + + # Enable OVN maintenance worker if using OVN + if [[ $Q_AGENT == "ovn" ]]; then + enable_service neutron-ovn-maintenance-worker + fi +} + # Start running processes function start_neutron_service_and_check { local service_port=$Q_PORT @@ -620,24 +640,15 @@ function start_neutron_service_and_check { service_protocol="http" fi - # Start the Neutron service - # The default value of "rpc_workers" is None (not defined). If - # "rpc_workers" is explicitly set to 0, the RPC workers process - # should not be executed. - local rpc_workers - rpc_workers=$(iniget_multiline $NEUTRON_CONF DEFAULT rpc_workers) + # Enable neutron server services based on configuration + enable_neutron_server_services - enable_service neutron-api + # Start the Neutron service processes run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" neutron_url=$Q_PROTOCOL://$Q_HOST/ - if [ "$rpc_workers" != "0" ]; then - enable_service neutron-rpc-server - fi - enable_service neutron-periodic-workers - _enable_ovn_maintenance - if [ "$rpc_workers" != "0" ]; then - run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $cfg_file_options" - fi + + # Start RPC server if enabled (run_process checks is_service_enabled internally) + run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $cfg_file_options" run_process neutron-periodic-workers "$NEUTRON_BIN_DIR/neutron-periodic-workers $cfg_file_options" _run_ovn_maintenance if [ ! -z "$NEUTRON_ENDPOINT_SERVICE_NAME" ]; then @@ -996,7 +1007,7 @@ function _configure_neutron_plugin_agent { # It is called when q-svc is enabled. function _configure_neutron_service { Q_API_PASTE_FILE=$NEUTRON_CONF_DIR/api-paste.ini - cp $NEUTRON_DIR/etc/api-paste.ini $Q_API_PASTE_FILE + cp $NEUTRON_DIR/etc/neutron/api-paste.ini $Q_API_PASTE_FILE # Update either configuration file with plugin iniset $NEUTRON_CONF DEFAULT core_plugin $Q_PLUGIN_CLASS @@ -1011,10 +1022,10 @@ function _configure_neutron_service { iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_status_changes $Q_NOTIFY_NOVA_PORT_STATUS_CHANGES iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_data_changes $Q_NOTIFY_NOVA_PORT_DATA_CHANGES - configure_keystone_authtoken_middleware $NEUTRON_CONF nova nova + configure_keystoneauth $NEUTRON_CONF nova nova # Configuration for placement client - configure_keystone_authtoken_middleware $NEUTRON_CONF placement placement + configure_keystoneauth $NEUTRON_CONF placement placement # Configure plugin neutron_plugin_configure_service @@ -1073,11 +1084,7 @@ function _neutron_setup_rootwrap { # Set up ``rootwrap.conf``, pointing to ``$NEUTRON_CONF_DIR/rootwrap.d`` # location moved in newer versions, prefer new location - if test -r $NEUTRON_DIR/etc/neutron/rootwrap.conf; then - sudo install -o root -g root -m 644 $NEUTRON_DIR/etc/neutron/rootwrap.conf $Q_RR_CONF_FILE - else - sudo install -o root -g root -m 644 $NEUTRON_DIR/etc/rootwrap.conf $Q_RR_CONF_FILE - fi + sudo install -o root -g root -m 644 $NEUTRON_DIR/etc/neutron/rootwrap.conf $Q_RR_CONF_FILE sudo sed -e "s:^filters_path=.*$:filters_path=$Q_CONF_ROOTWRAP_D:" -i $Q_RR_CONF_FILE # Rely on $PATH set by devstack to determine what is safe to execute # by rootwrap rather than use explicit whitelist of paths in diff --git a/lib/neutron_plugins/ml2 b/lib/neutron_plugins/ml2 index 687167bf79..71bede842e 100644 --- a/lib/neutron_plugins/ml2 +++ b/lib/neutron_plugins/ml2 @@ -72,16 +72,16 @@ function neutron_plugin_configure_common { function neutron_plugin_configure_service { if [[ "$Q_ML2_TENANT_NETWORK_TYPE" != "local" ]]; then - Q_SRV_EXTRA_OPTS+=(tenant_network_types=$Q_ML2_TENANT_NETWORK_TYPE) + Q_SRV_EXTRA_OPTS+=(project_network_types=$Q_ML2_TENANT_NETWORK_TYPE) elif [[ "$ENABLE_TENANT_TUNNELS" == "True" ]]; then # This assumes you want a simple configuration, and will overwrite # Q_SRV_EXTRA_OPTS if set in addition to ENABLE_TENANT_TUNNELS. - Q_SRV_EXTRA_OPTS+=(tenant_network_types=gre) + Q_SRV_EXTRA_OPTS+=(project_network_types=gre) Q_ML2_PLUGIN_GRE_TYPE_OPTIONS=(tunnel_id_ranges=$TENANT_TUNNEL_RANGES) elif [[ "$ENABLE_TENANT_VLANS" == "True" ]]; then - Q_SRV_EXTRA_OPTS+=(tenant_network_types=vlan) + Q_SRV_EXTRA_OPTS+=(project_network_types=vlan) else - echo "WARNING - The ml2 plugin is using local tenant networks, with no connectivity between hosts." + echo "WARNING - The ml2 plugin is using local project networks, with no connectivity between hosts." fi # Allow for overrding VLAN configuration (for example, to configure provider diff --git a/lib/neutron_plugins/openvswitch b/lib/neutron_plugins/openvswitch index 130eaacab3..c661a1a600 100644 --- a/lib/neutron_plugins/openvswitch +++ b/lib/neutron_plugins/openvswitch @@ -20,12 +20,12 @@ function neutron_plugin_configure_common { function neutron_plugin_configure_service { if [[ "$ENABLE_TENANT_TUNNELS" == "True" ]]; then - iniset /$Q_PLUGIN_CONF_FILE ovs tenant_network_type gre + iniset /$Q_PLUGIN_CONF_FILE ovs project_network_type gre iniset /$Q_PLUGIN_CONF_FILE ovs tunnel_id_ranges $TENANT_TUNNEL_RANGES elif [[ "$ENABLE_TENANT_VLANS" == "True" ]]; then - iniset /$Q_PLUGIN_CONF_FILE ovs tenant_network_type vlan + iniset /$Q_PLUGIN_CONF_FILE ovs project_network_type vlan else - echo "WARNING - The openvswitch plugin is using local tenant networks, with no connectivity between hosts." + echo "WARNING - The openvswitch plugin is using local project networks, with no connectivity between hosts." fi # Override ``OVS_VLAN_RANGES`` and ``OVS_BRIDGE_MAPPINGS`` in ``localrc`` diff --git a/lib/neutron_plugins/ovn_agent b/lib/neutron_plugins/ovn_agent index e58cd4fb38..8e3ca4ab89 100644 --- a/lib/neutron_plugins/ovn_agent +++ b/lib/neutron_plugins/ovn_agent @@ -96,8 +96,16 @@ OVN_META_CONF=$NEUTRON_CONF_DIR/neutron_ovn_metadata_agent.ini OVN_META_DATA_HOST=${OVN_META_DATA_HOST:-$(ipv6_unquote $SERVICE_HOST)} # OVN agent configuration +# The OVN agent is configured, by default, with the "metadata" extension. OVN_AGENT_CONF=$NEUTRON_CONF_DIR/plugins/ml2/ovn_agent.ini -OVN_AGENT_EXTENSIONS=${OVN_AGENT_EXTENSIONS:-} +OVN_AGENT_EXTENSIONS=${OVN_AGENT_EXTENSIONS:-metadata} +# The variable TARGET_ENABLE_OVN_AGENT, if True, overrides the OVN Metadata +# agent service (q-ovn-metadata-agent neutron-ovn-metadata-agent) and the OVN +# agent service (q-ovn-agent neutron-ovn-agent) configuration, always disabling +# the first one (OVN Metadata agent) and enabling the second (OVN agent). +# This variable will be removed in 2026.2, along with the OVN Metadata agent +# removal. +TARGET_ENABLE_OVN_AGENT=$(trueorfalse False TARGET_ENABLE_OVN_AGENT) # If True (default) the node will be considered a gateway node. ENABLE_CHASSIS_AS_GW=$(trueorfalse True ENABLE_CHASSIS_AS_GW) @@ -300,6 +308,21 @@ function create_public_bridge { _configure_public_network_connectivity } +function is_ovn_metadata_agent_enabled { + if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent && [[ "$TARGET_ENABLE_OVN_AGENT" == "False" ]]; then + return 0 + fi + return 1 +} + +function is_ovn_agent_enabled { + if is_service_enabled q-ovn-agent neutron-ovn-agent || [[ "$TARGET_ENABLE_OVN_AGENT" == "True" ]]; then + enable_service q-ovn-agent + return 0 + fi + return 1 + +} # OVN compilation functions # ------------------------- @@ -378,9 +401,6 @@ function install_ovn { # Check the OVN configuration ovn_sanity_check - # Install tox, used to generate the config (see devstack/override-defaults) - pip_install tox - sudo mkdir -p $OVS_RUNDIR sudo chown $(whoami) $OVS_RUNDIR @@ -406,9 +426,6 @@ function install_ovn { sudo mkdir -p $OVS_PREFIX/var/log/ovn sudo chown $(whoami) $OVS_PREFIX/var/log/ovn else - # Load fixup_ovn_centos - source ${TOP_DIR}/tools/fixup_stuff.sh - fixup_ovn_centos install_package $(get_packages openvswitch) install_package $(get_packages ovn) fi @@ -497,9 +514,9 @@ function configure_ovn_plugin { inicomment /$Q_PLUGIN_CONF_FILE network_log local_output_log_base="$Q_LOG_DRIVER_LOG_BASE" fi - if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then + if is_ovn_metadata_agent_enabled; then populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=True - elif is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then + elif is_ovn_agent_enabled && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=True else populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=False @@ -520,9 +537,9 @@ function configure_ovn_plugin { fi if is_service_enabled n-api-meta ; then - if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then + if is_ovn_metadata_agent_enabled; then iniset $NOVA_CONF neutron service_metadata_proxy True - elif is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then + elif is_ovn_agent_enabled && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then iniset $NOVA_CONF neutron service_metadata_proxy True fi fi @@ -557,10 +574,10 @@ function configure_ovn { # Metadata local sample_file="" local config_file="" - if is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]] && is_service_enabled ovn-controller; then + if is_ovn_agent_enabled && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]] && is_service_enabled ovn-controller; then sample_file=$NEUTRON_DIR/etc/neutron/plugins/ml2/ovn_agent.ini.sample config_file=$OVN_AGENT_CONF - elif is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent && is_service_enabled ovn-controller; then + elif is_ovn_metadata_agent_enabled && is_service_enabled ovn-controller; then sample_file=$NEUTRON_DIR/etc/neutron_ovn_metadata_agent.ini.sample config_file=$OVN_META_CONF fi @@ -757,13 +774,13 @@ function start_ovn { fi fi - if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then + if is_ovn_metadata_agent_enabled; then run_process q-ovn-metadata-agent "$NEUTRON_OVN_BIN_DIR/$NEUTRON_OVN_METADATA_BINARY --config-file $OVN_META_CONF" # Format logging setup_logging $OVN_META_CONF fi - if is_service_enabled q-ovn-agent neutron-ovn-agent; then + if is_ovn_agent_enabled; then run_process q-ovn-agent "$NEUTRON_OVN_BIN_DIR/$NEUTRON_OVN_AGENT_BINARY --config-file $OVN_AGENT_CONF" # Format logging setup_logging $OVN_AGENT_CONF @@ -778,20 +795,27 @@ function _stop_ovs_dp { function _stop_process { local service=$1 echo "Stopping process $service" - if $SYSTEMCTL is-enabled $service; then + # Stop if running, regardless of enabled state + if $SYSTEMCTL is-active $service; then $SYSTEMCTL stop $service + fi + if $SYSTEMCTL is-enabled $service; then $SYSTEMCTL disable $service fi } function stop_ovn { + # NOTE(ralonsoh): this check doesn't use "is_ovn_metadata_agent_enabled", + # instead it relies only in the configured services, disregarding the + # flag "TARGET_ENABLE_OVN_AGENT". It is needed to force the OVN Metadata + # agent stop in case the flag "TARGET_ENABLE_OVN_AGENT" is set. if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then # pkill takes care not to kill itself, but it may kill its parent # sudo unless we use the "ps | grep [f]oo" trick sudo pkill -9 -f "[h]aproxy" || : _stop_process "devstack@q-ovn-metadata-agent.service" fi - if is_service_enabled q-ovn-agent neutron-ovn-agent; then + if is_ovn_agent_enabled; then # pkill takes care not to kill itself, but it may kill its parent # sudo unless we use the "ps | grep [f]oo" trick sudo pkill -9 -f "[h]aproxy" || : @@ -810,10 +834,22 @@ function stop_ovn { _stop_process "devstack@ovs-vtep.service" fi + # Clear OVS external-ids before stopping to prevent stale config on restack + if sudo ovs-vsctl show &>/dev/null; then + sudo ovs-vsctl --if-exists clear open_vswitch . external-ids + fi + _stop_process "$OVS_VSWITCHD_SERVICE" _stop_process "$OVSDB_SERVER_SERVICE" _stop_ovs_dp + + # Clean up runtime files that can prevent restart + sudo rm -f $OVS_RUNDIR/*.sock $OVS_RUNDIR/*.pid $OVS_RUNDIR/*.ctl + sudo rm -f $OVN_RUNDIR/*.sock $OVN_RUNDIR/*.pid $OVN_RUNDIR/*.ctl + # Clean up database lock files + sudo rm -f $OVS_DATADIR/.*.db.~lock~ + sudo rm -f $OVN_DATADIR/.*.db.~lock~ } function _cleanup { diff --git a/lib/neutron_plugins/services/l3 b/lib/neutron_plugins/services/l3 index bbedc57a44..238dd34b56 100644 --- a/lib/neutron_plugins/services/l3 +++ b/lib/neutron_plugins/services/l3 @@ -35,7 +35,7 @@ Q_PUBLIC_VETH_INT=${Q_PUBLIC_VETH_INT:-veth-pub-int} # The next variable is configured by plugin # e.g. _configure_neutron_l3_agent or lib/neutron_plugins/* # -# L3 routers exist per tenant +# L3 routers exist per project Q_L3_ROUTER_PER_TENANT=${Q_L3_ROUTER_PER_TENANT:-True} @@ -216,7 +216,7 @@ function create_neutron_initial_network { if is_networking_extension_supported "router" && is_networking_extension_supported "external-net"; then # Create a router, and add the private subnet as one of its interfaces if [[ "$Q_L3_ROUTER_PER_TENANT" == "True" ]]; then - # create a tenant-owned router. + # create a project-owned router. ROUTER_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" router create $Q_ROUTER_NAME -f value -c id) die_if_not_set $LINENO ROUTER_ID "Failure creating router $Q_ROUTER_NAME" else @@ -387,7 +387,7 @@ function _neutron_configure_router_v6 { # Override global IPV6_ROUTER_GW_IP with the true value from neutron # NOTE(slaweq): when enforce scopes is enabled in Neutron, router's # gateway ports aren't visible in API because such ports don't belongs - # to any tenant. Because of that, at least temporary we need to find + # to any project. Because of that, at least temporary we need to find # IPv6 address of the router's gateway in a bit different way. # It can be reverted when bug # https://bugs.launchpad.net/neutron/+bug/1959332 will be fixed diff --git a/lib/nova b/lib/nova index 810a3d9554..8e0ea1756f 100644 --- a/lib/nova +++ b/lib/nova @@ -453,6 +453,10 @@ function create_nova_conf { iniset $NOVA_CONF DEFAULT metadata_listen "$NOVA_SERVICE_LISTEN_ADDRESS" iniset $NOVA_CONF DEFAULT shutdown_timeout $NOVA_SHUTDOWN_TIMEOUT + # Enable errors if response validation fails. We want this enabled in CI + # and development contexts to highlights bugs in our response schemas. + iniset $NOVA_CONF api response_validation error + iniset $NOVA_CONF key_manager backend nova.keymgr.conf_key_mgr.ConfKeyManager iniset $NOVA_CONF DEFAULT report_interval $NOVA_SERVICE_REPORT_INTERVAL @@ -628,32 +632,19 @@ function create_nova_conf { function configure_placement_nova_compute { # Use the provided config file path or default to $NOVA_CONF. local conf=${1:-$NOVA_CONF} - iniset $conf placement auth_type "password" - iniset $conf placement auth_url "$KEYSTONE_SERVICE_URI" - iniset $conf placement username nova - iniset $conf placement password "$SERVICE_PASSWORD" - iniset $conf placement user_domain_name "$SERVICE_DOMAIN_NAME" - iniset $conf placement project_name "$SERVICE_TENANT_NAME" - iniset $conf placement project_domain_name "$SERVICE_DOMAIN_NAME" - iniset $conf placement region_name "$REGION_NAME" + configure_keystoneauth $conf nova placement } # Configure access to cinder. function configure_cinder_access { iniset $NOVA_CONF cinder os_region_name "$REGION_NAME" - iniset $NOVA_CONF cinder auth_type "password" - iniset $NOVA_CONF cinder auth_url "$KEYSTONE_SERVICE_URI" # NOTE(mriedem): This looks a bit weird but we use the nova user here # since it has the admin role and the cinder user does not. This is # similar to using the nova user in init_nova_service_user_conf. We need # to use a user with the admin role for background tasks in nova to # be able to GET block-storage API resources owned by another project # since cinder has low-level "is_admin" checks in its DB API. - iniset $NOVA_CONF cinder username nova - iniset $NOVA_CONF cinder password "$SERVICE_PASSWORD" - iniset $NOVA_CONF cinder user_domain_name "$SERVICE_DOMAIN_NAME" - iniset $NOVA_CONF cinder project_name "$SERVICE_TENANT_NAME" - iniset $NOVA_CONF cinder project_domain_name "$SERVICE_DOMAIN_NAME" + configure_keystoneauth $conf nova cinder if is_service_enabled tls-proxy; then CINDER_SERVICE_HOST=${CINDER_SERVICE_HOST:-$SERVICE_HOST} CINDER_SERVICE_PORT=${CINDER_SERVICE_PORT:-8776} @@ -663,14 +654,7 @@ function configure_cinder_access { # Configure access to manila. function configure_manila_access { - iniset $NOVA_CONF manila os_region_name "$REGION_NAME" - iniset $NOVA_CONF manila auth_type "password" - iniset $NOVA_CONF manila auth_url "$KEYSTONE_SERVICE_URI" - iniset $NOVA_CONF manila username nova - iniset $NOVA_CONF manila password "$SERVICE_PASSWORD" - iniset $NOVA_CONF manila user_domain_name "$SERVICE_DOMAIN_NAME" - iniset $NOVA_CONF manila project_name "$SERVICE_TENANT_NAME" - iniset $NOVA_CONF manila project_domain_name "$SERVICE_DOMAIN_NAME" + configure_keystoneauth $conf nova manila } function configure_console_compute { @@ -836,14 +820,7 @@ function configure_nova_unified_limits { function init_nova_service_user_conf { iniset $NOVA_CONF service_user send_service_user_token True - iniset $NOVA_CONF service_user auth_type password - iniset $NOVA_CONF service_user auth_url "$KEYSTONE_SERVICE_URI" - iniset $NOVA_CONF service_user username nova - iniset $NOVA_CONF service_user password "$SERVICE_PASSWORD" - iniset $NOVA_CONF service_user user_domain_name "$SERVICE_DOMAIN_NAME" - iniset $NOVA_CONF service_user project_name "$SERVICE_PROJECT_NAME" - iniset $NOVA_CONF service_user project_domain_name "$SERVICE_DOMAIN_NAME" - iniset $NOVA_CONF service_user auth_strategy keystone + configure_keystoneauth $NOVA_CONF nova service_user } function conductor_conf { @@ -1086,6 +1063,15 @@ function start_nova_compute { # gets its own configuration and own log file. local fake_conf="${NOVA_FAKE_CONF}-${i}" iniset $fake_conf DEFAULT host "${HOSTNAME}${i}" + # Ensure that each fake compute has its own state path so that it + # can have its own compute_id file + local state_path + state_path="$NOVA_STATE_PATH/${HOSTNAME}${i}" + COMPUTE_ID=$(uuidgen) + sudo mkdir -p "$state_path" + iniset $fake_conf DEFAULT state_path "$state_path" + # use the generated UUID as the stable compute node UUID + echo "$COMPUTE_ID" | sudo tee "$state_path/compute_id" run_process "n-cpu-${i}" "$NOVA_BIN_DIR/nova-compute --config-file $NOVA_CPU_CONF --config-file $fake_conf" done else diff --git a/lib/nova_plugins/functions-libvirt b/lib/nova_plugins/functions-libvirt index 35840539da..7175931384 100644 --- a/lib/nova_plugins/functions-libvirt +++ b/lib/nova_plugins/functions-libvirt @@ -70,8 +70,9 @@ function install_libvirt { if is_ubuntu; then install_package qemu-system libvirt-clients libvirt-daemon-system libvirt-dev python3-libvirt systemd-coredump + install_package ovmf if is_arch "aarch64"; then - install_package qemu-efi + install_package qemu-efi-aarch64 fi #pip_install_gr elif is_fedora; then @@ -94,6 +95,7 @@ function install_libvirt { # pre-installed these install_package $qemu_package install_package libvirt libvirt-devel + install_package edk2-ovmf if [[ $DISTRO =~ rhel9 ]]; then pip_install_gr libvirt-python diff --git a/lib/nova_plugins/hypervisor-fake b/lib/nova_plugins/hypervisor-fake index 87ee49fa4b..39cb45ca67 100644 --- a/lib/nova_plugins/hypervisor-fake +++ b/lib/nova_plugins/hypervisor-fake @@ -36,7 +36,7 @@ function cleanup_nova_hypervisor { # configure_nova_hypervisor - Set config files, create data dirs, etc function configure_nova_hypervisor { - iniset $NOVA_CONF DEFAULT compute_driver "fake.FakeDriver" + iniset $NOVA_CONF DEFAULT compute_driver "fake.FakeDriverWithoutFakeNodes" # Disable arbitrary limits iniset $NOVA_CONF quota driver nova.quota.NoopQuotaDriver } diff --git a/lib/swift b/lib/swift index 862927437d..2710346451 100644 --- a/lib/swift +++ b/lib/swift @@ -434,6 +434,13 @@ function configure_swift { swift_pipeline+=" s3token" iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token auth_uri ${KEYSTONE_SERVICE_URI_V3} iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token delay_auth_decision true + iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token secret_cache_duration 900 + iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token auth_url ${KEYSTONE_SERVICE_URI} + iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token project_name 'service' + iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token project_domain_name ${SERVICE_DOMAIN_NAME} + iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token username 'swift' + iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token user_domain_name ${SERVICE_DOMAIN_NAME} + iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token password ${SERVICE_PASSWORD} fi swift_pipeline+=" keystoneauth" fi @@ -477,6 +484,9 @@ function configure_swift { iniset ${SWIFT_CONF_DIR}/swift.conf swift-constraints max_header_size ${SWIFT_MAX_HEADER_SIZE} iniset ${SWIFT_CONF_DIR}/swift.conf swift-constraints max_file_size ${SWIFT_MAX_FILE_SIZE} + # Create an additional storage policy + iniset ${SWIFT_CONF_DIR}/swift.conf storage-policy:1 name silver + local node_number for node_number in ${SWIFT_REPLICAS_SEQ}; do local swift_node_config=${SWIFT_CONF_DIR}/object-server/${node_number}.conf @@ -713,6 +723,10 @@ function init_swift { $SWIFT_BIN_DIR/swift-ring-builder object.builder rebalance 42 $SWIFT_BIN_DIR/swift-ring-builder container.builder rebalance 42 $SWIFT_BIN_DIR/swift-ring-builder account.builder rebalance 42 + + # An additional storage policy requires an object ring as well. + # Re-using the previously created one to use the same devices. + cp object.ring.gz object-1.ring.gz } && popd >/dev/null } @@ -838,6 +852,8 @@ function stop_swift { for type in proxy object container account; do stop_process s-${type} done + # Stop the container-sync daemon if it was started + stop_process s-container-sync # Blast out any stragglers pkill -f swift- || true } diff --git a/lib/tempest b/lib/tempest index c9486f6310..1ebe9c5f1f 100644 --- a/lib/tempest +++ b/lib/tempest @@ -105,6 +105,8 @@ TEMPEST_CONCURRENCY=${TEMPEST_CONCURRENCY:-$(nproc)} TEMPEST_FLAVOR_RAM=${TEMPEST_FLAVOR_RAM:-192} TEMPEST_FLAVOR_ALT_RAM=${TEMPEST_FLAVOR_ALT_RAM:-256} +TEMPEST_USE_ISO_IMAGE=$(trueorfalse False TEMPEST_USE_ISO_IMAGE) + # Functions # --------- @@ -161,12 +163,20 @@ function get_active_images { # start with a fresh array in case we are called multiple times img_array=() - while read -r IMAGE_NAME IMAGE_UUID; do + # NOTE(gmaan): Most of the iso image require ssh to be enabled explicitly + # and if we set those iso images in image_ref and image_ref_alt that can + # cause test to fail because many tests using image_ref and image_ref_alt + # to boot server also perform ssh. We skip to set iso image in tempest + # unless it is requested via TEMPEST_USE_ISO_IMAGE. + while read -r IMAGE_NAME IMAGE_UUID DISK_FORMAT; do + if [[ "$DISK_FORMAT" == "iso" ]] && [[ "$TEMPEST_USE_ISO_IMAGE" == False ]]; then + continue + fi if [ "$IMAGE_NAME" = "$DEFAULT_IMAGE_NAME" ]; then img_id="$IMAGE_UUID" fi img_array+=($IMAGE_UUID) - done < <(openstack --os-cloud devstack-admin image list --property status=active | awk -F'|' '!/^(+--)|ID|aki|ari/ { print $3,$2 }') + done < <(openstack --os-cloud devstack-admin image list --long --property status=active | awk -F'|' '!/^(+--)|ID|aki|ari/ { print $3,$2,$4 }') } function poll_glance_images { @@ -524,6 +534,10 @@ function configure_tempest { iniset $TEMPEST_CONFIG compute-feature-enabled serial_console True fi + # NOTE(gmaan): Since 2025.2, 'manager' role is available in nova. + local nova_policy_roles="admin,manager,member,reader,service" + iniset $TEMPEST_CONFIG compute-feature-enabled nova_policy_roles $nova_policy_roles + # Network iniset $TEMPEST_CONFIG network project_networks_reachable false iniset $TEMPEST_CONFIG network public_network_id "$public_network_id" diff --git a/roles/capture-system-logs/tasks/main.yaml b/roles/capture-system-logs/tasks/main.yaml index 77b5ec5098..4b5ec4836b 100644 --- a/roles/capture-system-logs/tasks/main.yaml +++ b/roles/capture-system-logs/tasks/main.yaml @@ -4,7 +4,13 @@ executable: /bin/bash cmd: | sudo iptables-save > {{ stage_dir }}/iptables.txt - df -h > {{ stage_dir }}/df.txt + + # NOTE(sfernand): Run 'df' with a 60s timeout to prevent hangs from + # stale NFS mounts. + timeout -s 9 60s df -h > {{ stage_dir }}/df.txt || true + # If 'df' times out, the mount output helps debug which NFS share + # is unresponsive. + mount > {{ stage_dir }}/mount.txt for py_ver in 2 3; do if [[ `which python${py_ver}` ]]; then diff --git a/roles/orchestrate-devstack/tasks/main.yaml b/roles/orchestrate-devstack/tasks/main.yaml index 2b8ae01a62..b8ee7e35a7 100644 --- a/roles/orchestrate-devstack/tasks/main.yaml +++ b/roles/orchestrate-devstack/tasks/main.yaml @@ -4,6 +4,7 @@ when: inventory_hostname == 'controller' - name: Setup devstack on sub-nodes + any_errors_fatal: true block: - name: Distribute the build sshkey for the user "stack" diff --git a/stack.sh b/stack.sh index 04b5f4ca6a..c6d37611c9 100755 --- a/stack.sh +++ b/stack.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash - # ``stack.sh`` is an opinionated OpenStack developer installation. It # installs and configures various combinations of **Cinder**, **Glance**, # **Horizon**, **Keystone**, **Nova**, **Neutron**, and **Swift** @@ -230,7 +229,7 @@ write_devstack_version # Warn users who aren't on an explicitly supported distro, but allow them to # override check and attempt installation with ``FORCE=yes ./stack`` -SUPPORTED_DISTROS="bookworm|jammy|noble|rhel9" +SUPPORTED_DISTROS="trixie|bookworm|noble|rhel9|rhel10" if [[ ! ${DISTRO} =~ $SUPPORTED_DISTROS ]]; then echo "WARNING: this script has not been tested on $DISTRO" @@ -302,16 +301,17 @@ function _install_epel { } function _install_rdo { - if [[ $DISTRO == "rhel9" ]]; then + if [[ $DISTRO =~ "rhel" ]]; then + VERSION=${DISTRO:4:2} rdo_release=${TARGET_BRANCH#*/} if [[ "$TARGET_BRANCH" == "master" ]]; then # adding delorean-deps repo to provide current master rpms - sudo wget https://trunk.rdoproject.org/centos9-master/delorean-deps.repo -O /etc/yum.repos.d/delorean-deps.repo + sudo wget https://trunk.rdoproject.org/centos${VERSION}-master/delorean-deps.repo -O /etc/yum.repos.d/delorean-deps.repo else if sudo dnf provides centos-release-openstack-${rdo_release} >/dev/null 2>&1; then sudo dnf -y install centos-release-openstack-${rdo_release} else - sudo wget https://trunk.rdoproject.org/centos9-${rdo_release}/delorean-deps.repo -O /etc/yum.repos.d/delorean-deps.repo + sudo wget https://trunk.rdoproject.org/centos${VERSION}-${rdo_release}/delorean-deps.repo -O /etc/yum.repos.d/delorean-deps.repo fi fi fi @@ -356,7 +356,7 @@ async_init # Certain services such as rabbitmq require that the local hostname resolves # correctly. Make sure it exists in /etc/hosts so that is always true. LOCAL_HOSTNAME=`hostname -s` -if ! fgrep -qwe "$LOCAL_HOSTNAME" /etc/hosts; then +if ! grep -Fqwe "$LOCAL_HOSTNAME" /etc/hosts; then sudo sed -i "s/\(^127.0.0.1.*\)/\1 $LOCAL_HOSTNAME/" /etc/hosts fi @@ -365,36 +365,7 @@ fi # to speed things up SKIP_EPEL_INSTALL=$(trueorfalse False SKIP_EPEL_INSTALL) -if [[ $DISTRO == "rhel8" ]]; then - # If we have /etc/ci/mirror_info.sh assume we're on a OpenStack CI - # node, where EPEL is installed (but disabled) and already - # pointing at our internal mirror - if [[ -f /etc/ci/mirror_info.sh ]]; then - SKIP_EPEL_INSTALL=True - sudo dnf config-manager --set-enabled epel - fi - - # PowerTools repo provides libyaml-devel required by devstack itself and - # EPEL packages assume that the PowerTools repository is enable. - sudo dnf config-manager --set-enabled PowerTools - - # CentOS 8.3 changed the repository name to lower case. - sudo dnf config-manager --set-enabled powertools - - if [[ ${SKIP_EPEL_INSTALL} != True ]]; then - _install_epel - fi - # Along with EPEL, CentOS (and a-likes) require some packages only - # available in RDO repositories (e.g. OVS, or later versions of - # kvm) to run. - _install_rdo - - # NOTE(cgoncalves): workaround RHBZ#1154272 - # dnf fails for non-privileged users when expired_repos.json doesn't exist. - # RHBZ: https://bugzilla.redhat.com/show_bug.cgi?id=1154272 - # Patch: https://github.com/rpm-software-management/dnf/pull/1448 - echo "[]" | sudo tee /var/cache/dnf/expired_repos.json -elif [[ $DISTRO == "rhel9" ]]; then +if [[ $DISTRO == "rhel9" ]]; then # for CentOS Stream 9 repository sudo dnf config-manager --set-enabled crb # for RHEL 9 repository @@ -408,6 +379,11 @@ elif [[ $DISTRO == "rhel9" ]]; then if is_package_installed curl-minimal; then sudo dnf swap -y curl-minimal curl fi +elif [[ $DISTRO == "rhel10" ]]; then + # for CentOS Stream 10 repository + sudo dnf config-manager --set-enabled crb + # rabbitmq and other packages are provided by RDO repositories. + _install_rdo elif [[ $DISTRO == "openEuler-22.03" ]]; then # There are some problem in openEuler. We should fix it first. Some required # package/action runs before fixup script. So we can't fix there. @@ -1005,6 +981,11 @@ if is_service_enabled tls-proxy; then fix_system_ca_bundle_path fi +if is_service_enabled cinder || [[ "$USE_CINDER_FOR_GLANCE" == "True" ]]; then + # os-brick setup required by glance, cinder, and nova + init_os_brick +fi + # Extras Install # -------------- diff --git a/stackrc b/stackrc index 0319fc8a50..a16080600b 100644 --- a/stackrc +++ b/stackrc @@ -75,7 +75,7 @@ if ! isset ENABLED_SERVICES ; then # OVN ENABLED_SERVICES+=,ovn-controller,ovn-northd,ovs-vswitchd,ovsdb-server # Neutron - ENABLED_SERVICES+=,q-svc,q-ovn-metadata-agent + ENABLED_SERVICES+=,q-svc,q-ovn-agent # Dashboard ENABLED_SERVICES+=,horizon # Additional services @@ -133,7 +133,7 @@ export PYTHON3_VERSION=${PYTHON3_VERSION:-${_DEFAULT_PYTHON3_VERSION:-3}} # Create a virtualenv with this # Use the built-in venv to avoid more dependencies -export VIRTUALENV_CMD="python3 -m venv" +export VIRTUALENV_CMD="python$PYTHON3_VERSION -m venv" # Default for log coloring is based on interactive-or-not. # Baseline assumption is that non-interactive invocations are for CI, @@ -252,7 +252,7 @@ REQUIREMENTS_DIR=${REQUIREMENTS_DIR:-$DEST/requirements} # Setting the variable to 'ALL' will activate the download for all # libraries. -DEVSTACK_SERIES="2025.2" +DEVSTACK_SERIES="2026.1" ############## # @@ -395,6 +395,10 @@ GITBRANCH["futurist"]=${FUTURIST_BRANCH:-$TARGET_BRANCH} GITREPO["debtcollector"]=${DEBTCOLLECTOR_REPO:-${GIT_BASE}/openstack/debtcollector.git} GITBRANCH["debtcollector"]=${DEBTCOLLECTOR_BRANCH:-$TARGET_BRANCH} +# etcd3gw library +GITREPO["etcd3gw"]=${ETCD3GW_REPO:-${GIT_BASE}/openstack/etcd3gw.git} +GITBRANCH["etcd3gw"]=${ETCD3GW_BRANCH:-$BRANCHLESS_TARGET_BRANCH} + # helpful state machines GITREPO["automaton"]=${AUTOMATON_REPO:-${GIT_BASE}/openstack/automaton.git} GITBRANCH["automaton"]=${AUTOMATON_BRANCH:-$TARGET_BRANCH} @@ -615,12 +619,13 @@ case "$VIRT_DRIVER" in LIBVIRT_TYPE=${LIBVIRT_TYPE:-kvm} LIBVIRT_CPU_MODE=${LIBVIRT_CPU_MODE:-custom} LIBVIRT_CPU_MODEL=${LIBVIRT_CPU_MODEL:-Nehalem} + + if [[ -z "$os_VENDOR" ]]; then + GetOSVersion + fi + if [[ "$os_VENDOR" =~ (Debian|Ubuntu) ]]; then - # The groups change with newer libvirt. Older Ubuntu used - # 'libvirtd', but now uses libvirt like Debian. Do a quick check - # to see if libvirtd group already exists to handle grenade's case. - LIBVIRT_GROUP=$(cut -d ':' -f 1 /etc/group | grep 'libvirtd$' || true) - LIBVIRT_GROUP=${LIBVIRT_GROUP:-libvirt} + LIBVIRT_GROUP=libvirt else LIBVIRT_GROUP=libvirtd fi @@ -705,12 +710,11 @@ fi EXTRA_CACHE_URLS="" # etcd3 defaults -ETCD_VERSION=${ETCD_VERSION:-v3.4.27} -ETCD_SHA256_AMD64=${ETCD_SHA256_AMD64:-"a32d21e006252dbc3405b0645ba8468021ed41376974b573285927bf39b39eb9"} -ETCD_SHA256_ARM64=${ETCD_SHA256_ARM64:-"ed7e257c225b9b9545fac22246b97f4074a4b5109676e92dbaebfb9315b69cc0"} -ETCD_SHA256_PPC64=${ETCD_SHA256_PPC64:-"eb8825e0bc2cbaf9e55947f5ee373ebc9ca43b6a2ea5ced3b992c81855fff37e"} -# etcd v3.2.x and later doesn't have anything for s390x -ETCD_SHA256_S390X=${ETCD_SHA256_S390X:-""} +ETCD_VERSION=${ETCD_VERSION:-v3.5.21} +ETCD_SHA256_AMD64=${ETCD_SHA256_AMD64:-"adddda4b06718e68671ffabff2f8cee48488ba61ad82900e639d108f2148501c"} +ETCD_SHA256_ARM64=${ETCD_SHA256_ARM64:-"95bf6918623a097c0385b96f139d90248614485e781ec9bee4768dbb6c79c53f"} +ETCD_SHA256_PPC64=${ETCD_SHA256_PPC64:-"6fb6ecb3d1b331eb177dc610a8efad3aceb1f836d6aeb439ba0bfac5d5c2a38c"} +ETCD_SHA256_S390X=${ETCD_SHA256_S390X:-"a211a83961ba8a7e94f7d6343ad769e699db21a715ba4f3b68cf31ea28f9c951"} # Make sure etcd3 downloads the correct architecture if is_arch "x86_64"; then ETCD_ARCH="amd64" @@ -722,15 +726,8 @@ elif is_arch "ppc64le"; then ETCD_ARCH="ppc64le" ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_PPC64} elif is_arch "s390x"; then - # An etcd3 binary for s390x is not available on github like it is - # for other arches. Only continue if a custom download URL was - # provided. - if [[ -n "${ETCD_DOWNLOAD_URL}" ]]; then - ETCD_ARCH="s390x" - ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_S390X} - else - exit_distro_not_supported "etcd3. No custom ETCD_DOWNLOAD_URL provided." - fi + ETCD_ARCH="s390x" + ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_S390X} else exit_distro_not_supported "invalid hardware type - $ETCD_ARCH" fi diff --git a/tests/test_ini_config.sh b/tests/test_ini_config.sh index 6367cde441..fd3896d6ba 100755 --- a/tests/test_ini_config.sh +++ b/tests/test_ini_config.sh @@ -47,6 +47,9 @@ multi = foo2 [fff] ampersand = +[ggg] +backslash = + [key_with_spaces] rgw special key = something @@ -88,7 +91,7 @@ fi # test iniget_sections VAL=$(iniget_sections "${TEST_INI}") -assert_equal "$VAL" "default aaa bbb ccc ddd eee fff key_with_spaces \ +assert_equal "$VAL" "default aaa bbb ccc ddd eee fff ggg key_with_spaces \ del_separate_options del_same_option del_missing_option \ del_missing_option_multi del_no_options" @@ -134,6 +137,16 @@ done VAL=$(iniget ${TEST_INI} fff ampersand) assert_equal "$VAL" "&y" "iniset ampersands in option" +# Test with backslash in value +iniset ${TEST_INI} ggg backslash 'foo\bar' +VAL=$(iniget ${TEST_INI} ggg backslash) +assert_equal "$VAL" 'foo\bar' "iniset backslash in value" + +# Test with both ampersand and backslash +iniset ${TEST_INI} ggg backslash 'foo\bar&baz' +VAL=$(iniget ${TEST_INI} ggg backslash) +assert_equal "$VAL" 'foo\bar&baz' "iniset ampersand and backslash in value" + # test empty option if ini_has_option ${SUDO_ARG} ${TEST_INI} ddd empty; then passed "ini_has_option: ddd.empty present" diff --git a/tests/test_libs_from_pypi.sh b/tests/test_libs_from_pypi.sh index 839e3a1328..9552c93c4f 100755 --- a/tests/test_libs_from_pypi.sh +++ b/tests/test_libs_from_pypi.sh @@ -45,7 +45,7 @@ ALL_LIBS+=" oslo.cache oslo.reports osprofiler cursive" ALL_LIBS+=" keystoneauth ironic-lib neutron-lib oslo.privsep" ALL_LIBS+=" diskimage-builder os-vif python-brick-cinderclient-ext" ALL_LIBS+=" castellan python-barbicanclient ovsdbapp os-ken os-resource-classes" -ALL_LIBS+=" oslo.limit" +ALL_LIBS+=" oslo.limit etcd3gw" # Generate the above list with # echo ${!GITREPO[@]} diff --git a/tests/test_meta_config.sh b/tests/test_meta_config.sh index 087aaf468b..30479f245a 100755 --- a/tests/test_meta_config.sh +++ b/tests/test_meta_config.sh @@ -137,6 +137,9 @@ foo=bar [some] random=config +[[test12|run_tests.sh/test.conf]] +foo=bar + [[test-multi-sections|test-multi-sections.conf]] [sec-1] cfg_item1 = abcd @@ -389,13 +392,12 @@ EXPECT_VAL=0 check_result "$VAL" "$EXPECT_VAL" set -e -echo -n "merge_config_group test10 not directory: " +echo -n "merge_config_group test10 create directory: " set +e -# function is expected to fail and exit, running it -# in a subprocess to let this script proceed -(merge_config_group test.conf test10) +STACK_USER=$(id -u -n) +merge_config_group test.conf test10 VAL=$? -EXPECT_VAL=255 +EXPECT_VAL=0 check_result "$VAL" "$EXPECT_VAL" set -e @@ -414,9 +416,21 @@ random = config non = sense' check_result "$VAL" "$EXPECT_VAL" +echo -n "merge_config_group test12 directory as file: " +set +e +# function is expected to fail and exit, running it +# in a subprocess to let this script proceed +(merge_config_group test.conf test12) +VAL=$? +EXPECT_VAL=255 +check_result "$VAL" "$EXPECT_VAL" +set -e + rm -f test.conf test1c.conf test2a.conf \ test-space.conf test-equals.conf test-strip.conf \ test-colon.conf test-env.conf test-multiline.conf \ test-multi-sections.conf test-same.conf rm -rf test-etc +rm -rf does-not-exist-dir + diff --git a/tools/fixup_stuff.sh b/tools/fixup_stuff.sh index faea44f1e0..9e2818f2cc 100755 --- a/tools/fixup_stuff.sh +++ b/tools/fixup_stuff.sh @@ -84,24 +84,11 @@ EOF # python3-setuptools RPM are deleted, it breaks some tools such as semanage # (used in diskimage-builder) that use the -s flag of the python # interpreter, enforcing the use of the packages from /usr/lib. - # Importing setuptools/pkg_resources in a such environment fails. + # Importing setuptools in a such environment fails. # Enforce the package re-installation to fix those applications. if is_package_installed python3-setuptools; then sudo dnf reinstall -y python3-setuptools fi - # Workaround CentOS 8-stream iputils and systemd Bug - # https://bugzilla.redhat.com/show_bug.cgi?id=2037807 - if [[ $os_VENDOR == "CentOSStream" && $os_RELEASE -eq 8 ]]; then - sudo sysctl -w net.ipv4.ping_group_range='0 2147483647' - fi -} - -function fixup_ovn_centos { - if [[ $os_VENDOR != "CentOS" ]]; then - return - fi - # OVN packages are part of this release for CentOS - yum_install centos-release-openstack-victoria } function fixup_ubuntu { diff --git a/tools/generate-devstack-plugins-list.sh b/tools/generate-devstack-plugins-list.sh index 3307943df9..45a926392e 100755 --- a/tools/generate-devstack-plugins-list.sh +++ b/tools/generate-devstack-plugins-list.sh @@ -38,6 +38,17 @@ # current working directory, it will be prepended or appended to # the generated reStructuredText plugins table respectively. +# Setup virtual environment +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VENV_DIR="${SCRIPT_DIR}/.venv" + +if [[ ! -d "${VENV_DIR}" ]]; then + python3 -m venv "${VENV_DIR}" +fi + +source "${VENV_DIR}/bin/activate" +pip install -q -r "${SCRIPT_DIR}/requirements.txt" + # Print the title underline for a RST table. Argument is the length # of the first column, second column is assumed to be "URL" function title_underline { @@ -54,7 +65,7 @@ if [[ -r data/devstack-plugins-registry.header ]]; then cat data/devstack-plugins-registry.header fi -sorted_plugins=$(python3 tools/generate-devstack-plugins-list.py) +sorted_plugins=$("${VENV_DIR}/bin/python3" tools/generate-devstack-plugins-list.py) # find the length of the name column & pad name_col_len=$(echo "${sorted_plugins}" | wc -L) diff --git a/tools/install_pip.sh b/tools/install_pip.sh index 91b180c06f..027693fc0a 100755 --- a/tools/install_pip.sh +++ b/tools/install_pip.sh @@ -38,7 +38,6 @@ FILES=$TOP_DIR/files # [1] https://opendev.org/openstack/project-config/src/branch/master/nodepool/elements/cache-devstack/source-repository-pip PIP_GET_PIP_URL=${PIP_GET_PIP_URL:-"https://bootstrap.pypa.io/get-pip.py"} -PIP_GET_PIP36_URL=${PIP_GET_PIP36_URL:-"https://bootstrap.pypa.io/pip/3.6/get-pip.py"} GetDistro echo "Distro: $DISTRO" @@ -57,14 +56,8 @@ function get_versions { function install_get_pip { - if [[ "$PYTHON3_VERSION" = "3.6" ]]; then - _pip_url=$PIP_GET_PIP36_URL - _local_pip="$FILES/$(basename $_pip_url)-py36" - else - _pip_url=$PIP_GET_PIP_URL - _local_pip="$FILES/$(basename $_pip_url)" - fi - + _pip_url=$PIP_GET_PIP_URL + _local_pip="$FILES/$(basename $_pip_url)" # If get-pip.py isn't python, delete it. This was probably an # outage on the server. @@ -127,7 +120,7 @@ if [[ -n $PYPI_ALTERNATIVE_URL ]]; then configure_pypi_alternative_url fi -if is_fedora && [[ ${DISTRO} == f* || ${DISTRO} == rhel9 ]]; then +if is_fedora && [[ ${DISTRO} == f* || ${DISTRO} == rhel* ]]; then # get-pip.py will not install over the python3-pip package in # Fedora 34 any more. # https://bugzilla.redhat.com/show_bug.cgi?id=1988935 diff --git a/tools/mlock_report.py b/tools/mlock_report.py index 1b081bbe6f..8cbda15895 100644 --- a/tools/mlock_report.py +++ b/tools/mlock_report.py @@ -6,7 +6,7 @@ LCK_SUMMARY_REGEX = re.compile( - "^VmLck:\s+(?P[\d]+)\s+kB", re.MULTILINE) + r"^VmLck:\s+(?P[\d]+)\s+kB", re.MULTILINE) def main(): diff --git a/tools/outfilter.py b/tools/outfilter.py index df03a779b5..3955d39794 100644 --- a/tools/outfilter.py +++ b/tools/outfilter.py @@ -26,8 +26,8 @@ import re import sys -IGNORE_LINES = re.compile('(set \+o|xtrace)') -HAS_DATE = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \|') +IGNORE_LINES = re.compile(r'(set \+o|xtrace)') +HAS_DATE = re.compile(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \|') def get_options(): diff --git a/tools/ping_neutron.sh b/tools/ping_neutron.sh index ab8e8dfca8..2b65fd0fb3 100755 --- a/tools/ping_neutron.sh +++ b/tools/ping_neutron.sh @@ -21,7 +21,7 @@ set -o pipefail TOP_DIR=$(cd $(dirname "$0")/.. && pwd) -# This *must* be run as the admin tenant +# This *must* be run as the admin project source $TOP_DIR/openrc admin admin function usage { @@ -29,7 +29,7 @@ function usage { ping_neutron.sh [ping args] This provides a wrapper to ping neutron guests that are on isolated -tenant networks that the caller can't normally reach. It does so by +project networks that the caller can't normally reach. It does so by using either the DHCP or Metadata network namespace to support both ML2/OVS and OVN. diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 0000000000..f2293605cf --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/tools/verify-ipv6-address.py b/tools/verify-ipv6-address.py new file mode 100644 index 0000000000..dc18fa6d8a --- /dev/null +++ b/tools/verify-ipv6-address.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import argparse +import ipaddress +import sys + +def main(): + parser = argparse.ArgumentParser( + description="Check if a given string is a valid IPv6 address.", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument( + "address", + help=( + "The IPv6 address string to validate.\n" + "Examples:\n" + " 2001:0db8:85a3:0000:0000:8a2e:0370:7334\n" + " 2001:db8::1\n" + " ::1\n" + " fe80::1%eth0 (scope IDs are handled)" + ), + ) + args = parser.parse_args() + + try: + # try to create a IPv6Address: if we fail to parse or get an + # IPv4Address then die + ip_obj = ipaddress.ip_address(args.address.strip('[]')) + if isinstance(ip_obj, ipaddress.IPv6Address): + sys.exit(0) + else: + sys.exit(1) + except ValueError: + sys.exit(1) + except Exception as e: + print(f"An unexpected error occurred during validation: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/verify-ipv6-only-deployments.sh b/tools/verify-ipv6-only-deployments.sh index 0f0cba8afe..a1acecbb3f 100755 --- a/tools/verify-ipv6-only-deployments.sh +++ b/tools/verify-ipv6-only-deployments.sh @@ -33,28 +33,23 @@ function verify_devstack_ipv6_setting { echo $TUNNEL_IP_VERSION "TUNNEL_IP_VERSION is not set to 6 so TUNNEL_ENDPOINT_IP cannot be an IPv6 address." exit 1 fi - is_service_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_host'"))') - if [[ "$is_service_host_ipv6" != "True" ]]; then + if ! python3 ${TOP_DIR}/tools/verify-ipv6-address.py "$_service_host"; then echo $SERVICE_HOST "SERVICE_HOST is not IPv6 which means devstack cannot deploy services on IPv6 addresses." exit 1 fi - is_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_host_ipv6'"))') - if [[ "$is_host_ipv6" != "True" ]]; then + if ! python3 ${TOP_DIR}/tools/verify-ipv6-address.py "$_host_ipv6"; then echo $HOST_IPV6 "HOST_IPV6 is not IPv6 which means devstack cannot deploy services on IPv6 addresses." exit 1 fi - is_service_listen_address=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_listen_address'"))') - if [[ "$is_service_listen_address" != "True" ]]; then + if ! python3 ${TOP_DIR}/tools/verify-ipv6-address.py "$_service_listen_address"; then echo $SERVICE_LISTEN_ADDRESS "SERVICE_LISTEN_ADDRESS is not IPv6 which means devstack cannot deploy services on IPv6 addresses." exit 1 fi - is_service_local_host=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_local_host'"))') - if [[ "$is_service_local_host" != "True" ]]; then + if ! python3 ${TOP_DIR}/tools/verify-ipv6-address.py "$_service_local_host"; then echo $SERVICE_LOCAL_HOST "SERVICE_LOCAL_HOST is not IPv6 which means devstack cannot deploy services on IPv6 addresses." exit 1 fi - is_tunnel_endpoint_ip=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_tunnel_endpoint_ip'"))') - if [[ "$is_tunnel_endpoint_ip" != "True" ]]; then + if ! python3 ${TOP_DIR}/tools/verify-ipv6-address.py "$_tunnel_endpoint_ip"; then echo $TUNNEL_ENDPOINT_IP "TUNNEL_ENDPOINT_IP is not IPv6 which means devstack will not deploy with an IPv6 endpoint address." exit 1 fi @@ -63,8 +58,7 @@ function verify_devstack_ipv6_setting { } function sanity_check_system_ipv6_enabled { - system_ipv6_enabled=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_ipv6_enabled())') - if [[ $system_ipv6_enabled != "True" ]]; then + if [ ! -f "/proc/sys/net/ipv6/conf/default/disable_ipv6" ] || [ "$(cat /proc/sys/net/ipv6/conf/default/disable_ipv6)" -ne "0" ]; then echo "IPv6 is disabled in system" exit 1 fi @@ -78,10 +72,8 @@ function verify_service_listen_address_is_ipv6 { for endpoint in ${endpoints}; do local endpoint_address='' endpoint_address=$(echo "$endpoint" | awk -F/ '{print $3}' | awk -F] '{print $1}') - endpoint_address=$(echo $endpoint_address | tr -d []) - local is_endpoint_ipv6='' - is_endpoint_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$endpoint_address'"))') - if [[ "$is_endpoint_ipv6" != "True" ]]; then + endpoint_address=$(echo $endpoint_address | tr -d '[]') + if ! python3 ${TOP_DIR}/tools/verify-ipv6-address.py "$endpoint_address"; then all_ipv6=False echo $endpoint ": This is not an IPv6 endpoint which means corresponding service is not listening on an IPv6 address." continue diff --git a/unstack.sh b/unstack.sh index 29c80718f8..1919ef8ad7 100755 --- a/unstack.sh +++ b/unstack.sh @@ -61,7 +61,6 @@ source $TOP_DIR/lib/tls # Source project function libraries source $TOP_DIR/lib/infra -source $TOP_DIR/lib/oslo source $TOP_DIR/lib/lvm source $TOP_DIR/lib/horizon source $TOP_DIR/lib/keystone @@ -74,6 +73,7 @@ source $TOP_DIR/lib/neutron source $TOP_DIR/lib/ldap source $TOP_DIR/lib/dstat source $TOP_DIR/lib/atop +source $TOP_DIR/lib/tcpdump source $TOP_DIR/lib/etcd3 # Extras Source @@ -88,6 +88,12 @@ fi load_plugin_settings +# Enable neutron server services so they can be properly stopped +# This replicates the service enabling logic from stack.sh +if is_service_enabled neutron; then + enable_neutron_server_services +fi + set -o xtrace # Run extras @@ -157,6 +163,11 @@ if [[ -n "$UNSTACK_ALL" ]]; then if is_service_enabled rabbit; then stop_service rabbitmq-server fi + + # Stop LDAP server + if is_service_enabled ldap; then + stop_ldap + fi fi if is_service_enabled neutron; then @@ -173,6 +184,10 @@ if is_service_enabled openstack-cli-server; then stop_service devstack@openstack-cli-server fi +if is_service_enabled tcpdump; then + stop_tcpdump +fi + stop_dstat if is_service_enabled atop; then