diff --git a/.gitreview b/.gitreview index 64aebd2..6bc1d6e 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/devstack-plugin-ceph.git +defaultbranch=unmaintained/2023.1 diff --git a/.zuul.yaml b/.zuul.yaml index 115194a..85068ca 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -2,9 +2,7 @@ name: devstack-plugin-ceph-tempest-py3-base abstract: true parent: tempest-full-py3 - # TODO: Remove the nodeset pinning to focal once below bug is fixed - # https://bugs.launchpad.net/devstack-plugin-ceph/+bug/1996628 - nodeset: openstack-single-node-focal + nodeset: openstack-single-node-jammy description: | Base integration tests that runs with the ceph devstack plugin and py3. Former names for this job where: @@ -21,11 +19,14 @@ - ^tox.ini$ timeout: 7200 vars: + configure_swap_size: 8192 + tempest_concurrency: 3 devstack_localrc: ENABLE_FILE_INJECTION: false TEMPEST_PLUGINS: '/opt/stack/cinder-tempest-plugin' ENABLE_VOLUME_MULTIATTACH: false - TEMPEST_RUN_VALIDATION: false + TEMPEST_RUN_VALIDATION: True + MYSQL_REDUCE_MEMORY: True devstack_plugins: devstack-plugin-ceph: https://opendev.org/openstack/devstack-plugin-ceph devstack_services: @@ -45,19 +46,8 @@ This job enable the multiattach feature enable from stein on. vars: devstack_localrc: - ENABLE_VOLUME_MULTIATTACH: true - CEPH_RELEASE: "pacific" - -- job: - name: devstack-plugin-ceph-tempest-cephadm - parent: devstack-plugin-ceph-tempest-py3-base - description: | - Integration tests that runs with the ceph devstack plugin and py3. - The ceph cluster is deployed using cephadm - vars: - tempest_concurrency: 1 - devstack_localrc: - CEPHADM_DEPLOY: true + DISABLE_CEPHADM_POST_DEPLOY: True + CEPHADM_DEPLOY: True - job: name: devstack-plugin-ceph-compute-local-ephemeral @@ -85,42 +75,43 @@ description: | Runs manila tempest plugin tests with Native CephFS as a manila back end (DHSS=False) - parent: manila-tempest-plugin-cephfs-native + parent: manila-tempest-plugin-cephfs-native-cephadm - job: - name: devstack-plugin-ceph-cephfs-nfs + name: devstack-plugin-ceph-cephfs-nfs-standalone description: | Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila - back end (DHSS=False) + back end (DHSS=False). The Ceph cluster is created with cephadm + while nfs-ganesha is installed "standalone" via a package. parent: manila-tempest-plugin-cephfs-nfs - job: - name: devstack-plugin-ceph-tempest-fedora-latest - parent: devstack-plugin-ceph-tempest-py3 - description: | - Integration tests that runs with the ceph devstack plugin on Fedora. - nodeset: devstack-single-node-fedora-latest - voting: false + name: devstack-plugin-ceph-multinode-cephfs-nfs-cephadm + parent: manila-tempest-plugin-multinode-cephfs-nfs-cephadm + description: Test CephFS NFS (DHSS=False) in a Multinode devstack env - job: name: devstack-plugin-ceph-multinode-tempest-py3 parent: tempest-multinode-full-py3 description: | Integration tests that runs the ceph device plugin across multiple - nodes on py3. - # TODO: Remove the nodeset pinning to focal once below bug is fixed - # https://bugs.launchpad.net/devstack-plugin-ceph/+bug/1996628 - nodeset: openstack-two-node-focal + nodes on py3. The Ceph deployment strategy used by this job is Cephadm. required-projects: - openstack/cinder-tempest-plugin - openstack/devstack-plugin-ceph timeout: 10800 - voting: false vars: + configure_swap_size: 8192 + tempest_concurrency: 3 devstack_localrc: ENABLE_FILE_INJECTION: false ENABLE_VOLUME_MULTIATTACH: true - TEMPEST_RUN_VALIDATION: false + TEMPEST_RUN_VALIDATION: true + USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION: false + CEPHADM_DEPLOY: True + DISABLE_CEPHADM_POST_DEPLOY: True + MYSQL_REDUCE_MEMORY: True + REMOTE_CEPH: False CINDER_CEPH_UUID: d531d2d4-3937-429c-b0c2-658fe41e82aa devstack_plugins: devstack-plugin-ceph: https://opendev.org/openstack/devstack-plugin-ceph @@ -136,7 +127,8 @@ group-vars: subnode: devstack_localrc: - REMOTE_CEPH: true + REMOTE_CEPH: True + CEPHADM_DEPLOY: True CINDER_CEPH_UUID: d531d2d4-3937-429c-b0c2-658fe41e82aa - job: @@ -150,19 +142,6 @@ devstack_localrc: TEST_MASTER: true -- job: - name: devstack-plugin-ceph-multinode-tempest-cephadm - parent: devstack-plugin-ceph-multinode-tempest-py3 - description: | - Integration tests that runs the ceph device plugin across multiple - nodes on py3. - The ceph deployment strategy used by this job is cephadm. - vars: - devstack_localrc: - USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION: false - CEPHADM_DEPLOY: true - tempest_concurrency: 1 - - project-template: name: devstack-plugin-ceph-tempest-jobs description: | @@ -171,23 +150,23 @@ jobs: - devstack-plugin-ceph-tempest-py3: voting: false - - devstack-plugin-ceph-tempest-cephadm: - voting: false + - devstack-plugin-ceph-multinode-tempest-py3 + - devstack-plugin-ceph-multinode-cephfs-nfs-cephadm - devstack-plugin-ceph-cephfs-native: irrelevant-files: *irrelevant-files voting: false - # - devstack-plugin-ceph-cephfs-nfs: - # irrelevant-files: *irrelevant-files - # voting: false + - devstack-plugin-ceph-cephfs-nfs-standalone: + irrelevant-files: *irrelevant-files + voting: false # - devstack-plugin-ceph-tempest-fedora-latest # - devstack-plugin-ceph-multinode-tempest-py3 # - devstack-plugin-ceph-multinode-tempest-cephadm: # voting: false # - devstack-plugin-ceph-master-tempest: # voting: false - # gate: - # jobs: - # - devstack-plugin-ceph-tempest-py3 + gate: + jobs: + - devstack-plugin-ceph-tempest-py3 - project: templates: diff --git a/devstack/files/debs/devstack-plugin-ceph b/devstack/files/debs/devstack-plugin-ceph new file mode 100644 index 0000000..73bc9b9 --- /dev/null +++ b/devstack/files/debs/devstack-plugin-ceph @@ -0,0 +1,6 @@ +xfsprogs +qemu-block-extra +catatonit +podman +jq +ceph-common diff --git a/devstack/files/rpms/devstack-plugin-ceph b/devstack/files/rpms/devstack-plugin-ceph new file mode 100644 index 0000000..1b806da --- /dev/null +++ b/devstack/files/rpms/devstack-plugin-ceph @@ -0,0 +1,5 @@ +xfsprogs +dbus-tools +podman +jq +ceph-common diff --git a/devstack/lib/ceph b/devstack/lib/ceph index 9b37b61..7e15427 100755 --- a/devstack/lib/ceph +++ b/devstack/lib/ceph @@ -21,6 +21,7 @@ # Save trace setting XTRACE=$(set +o | grep xtrace) set +o xtrace +source $CEPH_PLUGIN_DIR/lib/common # Defaults @@ -30,8 +31,6 @@ TEST_MASTER=$(trueorfalse False TEST_MASTER) CEPH_RELEASE=${CEPH_RELEASE:-pacific} -GANESHA_RELEASE=${GANESHA_RELEASE:-V3.5-stable} - # Deploy a Ceph demo container instead of a non-containerized version CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED) @@ -99,10 +98,6 @@ CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data} MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative} MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} -# Allows driver to store NFS-Ganesha exports and export counter as -# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later, -# Ceph v12.2.2 or later, and OpenStack Queens or later. -MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True} # Set ``CEPH_REPLICAS`` to configure how many replicas are to be # configured for your Ceph cluster. By default we are configuring @@ -273,17 +268,13 @@ function _undefine_virsh_secret { # check_os_support_ceph() - Check if the OS provides a decent version of Ceph function check_os_support_ceph { - if [[ ! ${DISTRO} =~ (jammy|focal|bionic|xenial|f31|f32|f33|f34|rhel8) ]]; then - echo "WARNING: your distro $DISTRO does not provide \ - (at least) the Luminous release. \ - Please use Ubuntu Xenial, Ubuntu Bionic, Ubuntu Focal, Ubuntu Jammy, \ - Fedora 31-34 or CentOS Stream 8." + if [[ ! ${DISTRO} =~ (jammy|focal|bionic|xenial|f31|f32|f33|f34|rhel8|rhel9) ]]; then + echo "WARNING: devstack-plugin-ceph hasn't been tested with $DISTRO. \ + Set FORCE_CEPH_INSTALL=yes in your local.conf if you'd like to \ + attempt installation anyway." if [[ "$FORCE_CEPH_INSTALL" != "yes" ]]; then - die $LINENO "If you wish to install Ceph on this distribution \ - anyway run with FORCE_CEPH_INSTALL=yes, \ - this assumes that YOU will setup the proper repositories" + die $LINENO "Not proceeding with install." fi - NO_UPDATE_REPOS=False fi if [[ ! $INIT_SYSTEM == 'systemd' ]]; then @@ -749,67 +740,13 @@ function configure_ceph_manila { if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then configure_nfs_ganesha - # NFS-Ganesha server cannot run alongwith with other kernel NFS server. - sudo systemctl stop nfs-server || true - sudo systemctl disable nfs-server || true - sudo systemctl enable nfs-ganesha - sudo systemctl start nfs-ganesha || ( - echo "Ganesha didn't start. Let's debug..." >&2 - sudo systemctl status nfs-ganesha || true - echo "**Ganesha conf file**" >&2 - sudo cat /etc/ganesha/ganesha.conf || true - echo "**Ganesha log file**" >&2 - sudo cat /var/log/ganesha/ganesha.log || true - echo "**Exiting**" >&2 - exit 1 - ) - echo "Ganesha started successfully!" >&2 + start_nfs_ganesha fi # RESTART DOCKER CONTAINER } -function configure_nfs_ganesha { - # Configure NFS-Ganesha to work with Manila's CephFS driver - sudo mkdir -p /etc/ganesha/export.d - if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then - # Create an empty placeholder ganesha export index object - echo | sudo rados -p ${CEPHFS_DATA_POOL} put ganesha-export-index - - cat </dev/null -RADOS_URLS { - ceph_conf = ${CEPH_CONF_FILE}; - userid = admin; -} - -CACHEINODE { - Dir_Max = 1; - Dir_Chunk = 0; - - Cache_FDs = false; - - NParts = 1; - Cache_Size = 1; -} - -EXPORT_DEFAULTS { - Attr_Expiration_Time = 0; -} - -%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index -EOF - else - sudo touch /etc/ganesha/export.d/INDEX.conf - echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf - fi -} - -function cleanup_nfs_ganesha { - sudo systemctl stop nfs-ganesha - sudo systemctl disable nfs-ganesha - sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs -} - function configure_ceph_embedded_manila { if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \ @@ -893,7 +830,7 @@ function configure_ceph_cinder { sudo rbd pool init ${CINDER_CEPH_POOL} } -# install_ceph() - Collect source and prepare +# install_ceph_remote() - Collect source and prepare function install_ceph_remote { install_package ceph-common # ceph-common in Bionic (18.04) installs only the python2 variants of @@ -920,12 +857,12 @@ function install_ceph_remote { function dnf_add_repository_ceph { local ceph_release=$1 - local distro_release=$2 + local package_release=$2 cat > ceph.repo < nfs-ganesha.repo < \ -# [] -# - package_manager: apt-get or yum -# - ceph_release: luminous, ... -# - distro_release: 7, xenial, bionic +# Usage: configure_repo_ceph +# - package_release: to override the os_RELEASE variable function configure_repo_ceph { - local package_manager="$1" - local ceph_release="$2" - local distro_release="$3" + + package_release=${1:-$os_RELEASE} if is_ubuntu; then if [[ "${TEST_MASTER}" == "True" ]]; then repo_file_name="/etc/apt/sources.list.d/ceph-master.list" - sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/ubuntu/${distro_release}/flavors/default/repo" -O ${repo_file_name} + sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/ubuntu/${package_release}/flavors/default/repo" -O ${repo_file_name} else wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add - - sudo apt-add-repository -y "deb https://download.ceph.com/debian-${ceph_release}/ ${distro_release} main" + sudo apt-add-repository -y "deb https://download.ceph.com/debian-${CEPH_RELEASE}/ $package_release main" fi - sudo ${package_manager} -y update + sudo apt-get -y update elif is_fedora; then + package_release="el"${package_release} if [[ "${TEST_MASTER}" == "True" ]]; then repo_file_name="/etc/yum.repos.d/ceph-master.repo" - sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/centos/${distro_release}/flavors/default/repo" -O ${repo_file_name} + sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/centos/${package_release}/flavors/default/repo" -O ${repo_file_name} sudo dnf config-manager --add-repo ${repo_file_name} else - dnf_add_repository_ceph ${ceph_release} ${distro_release} + dnf_add_repository_ceph ${CEPH_RELEASE} ${package_release} fi fi } @@ -1023,74 +925,11 @@ function cleanup_repo_ceph { fi } -# configure_repo_nfsganesha() - Configure NFS Ganesha repositories -# Usage: configure_repo_nfsganesha \ -# [] -# - package_manager: apt-get or dnf -# - ganesha_release: 2.7, 2.8, 3.0 -# - ceph_release: ceph_luminous, ceph_nautilus -function configure_repo_nfsganesha { - local package_manager="$1" - local ganesha_release="$2" - local ceph_release="$3" - - if is_ubuntu; then - # FIXME(vkmc) We need to use community ppa's because there are no builds - # for ubuntu bionic and ubuntu focal available for nfs-ganesha 2.7 and above - # Remove this when they provide the build in download.ceph.com - # FIXME(vkmc) Community ppa's don't provide builds for nfs-ganesha-3 - # microversions (3.3, 3.5). Default to latest. - if [[ $GANESHA_RELEASE =~ V3.(3|5)-stable ]]; then - sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0 - sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0 - elif [ $GANESHA_RELEASE == 'V2.8-stable' ]; then - sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-1.8 - sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-2.8 - elif [ $GANESHA_RELEASE == 'V2.7-stable' ]; then - sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-1.7 - sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-2.7 - else - die $LINENO "GANESHA_RELEASE is not supported by the Ceph plugin for Devstack" - fi - sudo ${package_manager} -y update - elif is_fedora; then - dnf_add_repository_nfsganesha rpm-${ganesha_release} ${ceph_release} - fi -} - -# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories -# Usage: cleanup_repo_nfsganesha -function cleanup_repo_nfsganesha { - if is_ubuntu; then - # FIXME(vkmc) We need to use community ppa's because there are no builds - # for ubuntu bionic and ubuntu focal available for nfs-ganesha 2.7 and above - # Remove this when they provide the builds in download.ceph.com - # FIXME(vkmc) Community ppa's don't provide builds for nfs-ganesha-3 - # microversions (3.3, 3.5). Default to latest. - if [[ $GANESHA_RELEASE =~ V3.(3|5)-stable ]]; then - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-3_0-*.list* - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-3_0-*.list* - elif [ $GANESHA_RELEASE == 'V2.8-stable' ]; then - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-1_8-*.list* - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-2_8-*.list* - elif [ $GANESHA_RELEASE == 'V2.7-stable' ]; then - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-1_7-*.list* - sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-2_7-*.list* - else - die $LINENO "GANESHA_RELEASE is not supported by the Ceph plugin for Devstack" - fi - elif is_fedora; then - sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo - fi -} - function setup_packages_for_manila_on_ubuntu { CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2" if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - configure_repo_nfsganesha "apt-get" "$GANESHA_RELEASE" "$CEPH_RELEASE" - CEPH_PACKAGES="${CEPH_PACKAGES} libntirpc3 nfs-ganesha nfs-ganesha-ceph \ - nfs-ganesha-rados-urls nfs-ganesha-vfs" + install_nfs_ganesha fi if python3_enabled; then @@ -1100,31 +939,16 @@ function setup_packages_for_manila_on_ubuntu { function setup_packages_for_manila_on_fedora_family { if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - # NOTE(vkmc) el7 packages work on Fedora - configure_repo_nfsganesha "dnf" "$GANESHA_RELEASE" "$CEPH_RELEASE" - CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \ - nfs-ganesha-rados-urls nfs-ganesha-vfs" + install_nfs_ganesha fi } function install_ceph { if is_ubuntu; then - if ! [[ $os_CODENAME =~ (jammy|focal|xenial|bionic) ]]; then - die $LINENO "Supported for Ubuntu Xenial, Bionic, Focal or Jammy. \ - Not supported for other releases." - fi - # NOTE(vkmc) Dependencies for setting up repos install_package software-properties-common - # NOTE(noonedeadpunk): There're no community repos for Ubuntu Jammy yet - # but Ceph Quincy is provided from default - # Ubuntu 22.04 repos. - if ! [[ $os_CODENAME =~ (jammy) ]]; then - configure_repo_ceph "apt-get" "$CEPH_RELEASE" "$os_CODENAME" - fi - CEPH_PACKAGES="ceph libnss3-tools" if python3_enabled; then CEPH_PACKAGES="${CEPH_PACKAGES} python3-rados python3-rbd" @@ -1150,12 +974,14 @@ function install_ceph { install_package ${CEPH_PACKAGES} elif is_fedora; then + override_os_release="" if ! [[ $os_VENDOR =~ Fedora ]] && [[ $os_RELEASE =~ (31|32) ]]; then die $LINENO "Supported for Fedora 31 and 32. Not supported for other releases." + override_os_release="8" fi # NOTE(lyarwood) Use the py3 el8 packages on Fedora - configure_repo_ceph "dnf" "$CEPH_RELEASE" "el8" + configure_repo_ceph ${override_os_release} CEPH_PACKAGES="ceph" @@ -1230,8 +1056,7 @@ function stop_ceph { fi if is_ceph_enabled_for_service manila; then if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then - sudo systemctl stop nfs-ganesha - sudo systemctl disable nfs-ganesha + stop_nfs_ganesha fi sudo systemctl stop ceph-mds@${MDS_ID} sudo systemctl disable ceph-mds@${MDS_ID} diff --git a/devstack/lib/cephadm b/devstack/lib/cephadm index 8b5e685..7f5c5e3 100755 --- a/devstack/lib/cephadm +++ b/devstack/lib/cephadm @@ -18,7 +18,7 @@ XTRACE=$(set +o | grep xtrace) set +o xtrace # GENERIC CEPHADM INTERNAL OPTIONS, DO NOT EDIT -CEPH_RELEASE=${CEPH_RELEASE:-quincy} +CEPH_RELEASE=${CEPH_RELEASE:-reef} CEPH_PUB_KEY="/etc/ceph/ceph.pub" CEPH_CONFIG="/etc/ceph/ceph.conf" BOOTSTRAP_CONFIG="$HOME/bootstrap_ceph.conf" @@ -29,9 +29,13 @@ DISABLE_CEPHADM_POST_DEPLOY=${DISABLE_CEPHADM_POST_DEPLOY:-False} # DEFAULT OPTIONS ATTEMPTS=30 -CONTAINER_IMAGE=${CONTAINER_IMAGE:-'quay.io/ceph/ceph:v17.2.3'} +CONTAINER_IMAGE=${CONTAINER_IMAGE:-'quay.io/ceph/ceph:v18.2'} DEVICES=() -FSID=$(uuidgen) +if [[ "$REMOTE_CEPH" = "False" ]]; then + FSID=$(uuidgen) +else + FSID=$(cat ceph.conf | awk '/fsid/ { print $3 }') +fi KEY_EXPORT_DIR="/etc/ceph" KEYS=("client.openstack") # at least the client.openstack default key should be created MIN_OSDS=1 @@ -61,7 +65,11 @@ RBD_CLIENT_LOG=/var/log/ceph/qemu-guest-\$pid.log # MANILA DEFAULTS MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} -# NFS OPTIONS +# NFS OPTIONS: Only apply when ENABLE_CEPH_MANILA=True +# Whether or not cephadm should deploy/manage NFS-Ganesha? If set to False, +# we'll deploy a "standalone" NFS Ganesha instead, not managed by cephadm. +CEPHADM_DEPLOY_NFS=${CEPHADM_DEPLOY_NFS:-True} +# Clustered NFS Options FSNAME=${FSNAME:-'cephfs'} NFS_PORT=2049 CEPHFS_CLIENT=0 @@ -117,33 +125,19 @@ function export_spec { echo "Ceph cluster config exported: $EXPORT" } -# Pre-install ceph: install podman -function _install_podman { - # FIXME(vkmc) Check required for Ubuntu 20.04 LTS (current CI node) - # Remove when our CI is pushed to the next LTS version - if ! command -v podman &> /dev/null; then - if [[ $os_CODENAME =~ (focal) ]]; then - echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /" \ - | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list - curl -L "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/Release.key" \ - | sudo apt-key add - - sudo apt-get update - sudo apt-get -y upgrade - fi - install_package podman - fi -} - # Pre-install ceph: install required dependencies function install_deps { - install_package jq ceph-common - _install_podman - install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests + if [[ "$REMOTE_CEPH" = "False" ]]; then + install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests + fi } # Pre-install ceph: get cephadm binary function get_cephadm { - curl -O https://raw.githubusercontent.com/ceph/ceph/"$CEPH_RELEASE"/src/cephadm/cephadm + # NOTE(gouthamr): cephadm binary here is a python executable, and the + # $os_PACKAGE ("rpm") or $os_release (el9) doesn't really matter. There is + # no ubuntu/debian equivalent being published by the ceph community. + curl -O https://download.ceph.com/rpm-${CEPH_RELEASE}/el9/noarch/cephadm $SUDO mv cephadm $TARGET_BIN $SUDO chmod +x $TARGET_BIN/cephadm echo "[GET CEPHADM] cephadm is now available" @@ -176,7 +170,7 @@ EOF function start_ceph { cluster=$(sudo cephadm ls | jq '.[]' | jq 'select(.name | test("^mon*")).fsid') if [ -z "$cluster" ]; then - $SUDO $CEPHADM --image "$CONTAINER_IMAGE" \ + $SUDO "$CEPHADM" --image "$CONTAINER_IMAGE" \ bootstrap \ --fsid $FSID \ --config "$BOOTSTRAP_CONFIG" \ @@ -192,16 +186,16 @@ function start_ceph { --skip-mon-network \ --mon-ip "$HOST_IP" - test -e $CEPH_CONFIG - test -e $CEPH_KEYRING + test -e $CEPH_CONFIG + test -e $CEPH_KEYRING - if [ "$CEPHADM_DEV_OSD" == 'True' ]; then - create_osd_dev - fi - # Wait cephadm backend to be operational - # and add osds via drivegroups - sleep "$SLEEP" - add_osds + if [ "$CEPHADM_DEV_OSD" == 'True' ]; then + create_osd_dev + fi + # Wait cephadm backend to be operational + # and add osds via drivegroups + sleep "$SLEEP" + add_osds fi } @@ -234,7 +228,7 @@ function add_osds { # let's add some osds if [ -z "$DEVICES" ]; then echo "Using ALL available devices" - $SUDO $CEPHADM shell ceph orch apply osd --all-available-devices + $SUDO "$CEPHADM" shell ceph orch apply osd --all-available-devices else for item in "${DEVICES[@]}"; do echo "Creating osd $item on node $HOSTNAME" @@ -244,8 +238,8 @@ function add_osds { fi while [ "$ATTEMPTS" -ne 0 ]; do - num_osds=$($SUDO $CEPHADM shell --fsid $FSID --config $CEPH_CONFIG \ - --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds') + num_osds=$($SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ + --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds') if [ "$num_osds" -ge "$MIN_OSDS" ]; then break; fi @@ -300,6 +294,7 @@ function _create_key { --keyring $CEPH_KEYRING -- ceph auth get-or-create "$name" mgr "allow rw" mon "allow r" osd "$osd_caps" \ -o "$KEY_EXPORT_DIR/ceph.$name.keyring" + $SUDO chown ${STACK_USER}:$(id -g -n $whoami) \ ${CEPH_CONF_DIR}/ceph.$name.keyring } @@ -315,22 +310,50 @@ function create_keys { # Install ceph: add MDS function cephfs_config { # Two pools are generated by this action - # - $FSNAME.FSNAME.data - # - $FSNAME.FSNAME.meta + # - cephfs.$FSNAME.data + # - cephfs.$FSNAME.meta # and the mds daemon is deployed $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph fs volume create "$FSNAME" } -# Install ceph: add NFS -function ceph_nfs_config { - # (fpantano) TODO: Build an ingress daemon on top of this +# Get Ceph version +function _get_ceph_version { + local ceph_version_str + + ceph_version_str=$(sudo podman run --rm --entrypoint ceph $CONTAINER_IMAGE \ + --version | awk '{ print $3 }') + + echo $ceph_version_str +} + +function _install_and_configure_clustered_nfs { + local ceph_version + ceph_version=$(_get_ceph_version) + echo "[CEPHADM] Deploy nfs.$FSNAME backend" $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ --keyring $CEPH_KEYRING -- ceph orch apply nfs \ "$FSNAME" --placement="$HOSTNAME" --port $NFS_PORT } +function _install_and_configure_standalone_nfs { + source $CEPH_PLUGIN_DIR/lib/common + install_nfs_ganesha + configure_nfs_ganesha + start_nfs_ganesha +} + +# Install ceph: add NFS +function ceph_nfs_config { + if [[ "$CEPHADM_DEPLOY_NFS" == "True" ]]; then + _install_and_configure_clustered_nfs + else + _install_and_configure_standalone_nfs + fi + +} + function _create_swift_endpoint { local swift_service @@ -434,17 +457,17 @@ function configure_ceph_manila { function enable_services { for item in "${SERVICES[@]}"; do case "$item" in - cephfs|CEPHFS) + cephfs|CEPHFS) echo "[CEPHADM] Config cephfs volume on node $HOSTNAME" cephfs_config CEPHFS_CLIENT=1 ;; - nfs|NFS) + nfs|NFS) echo "[CEPHADM] Deploying NFS on node $HOSTNAME" ceph_nfs_config CEPHFS_CLIENT=1 ;; - rgw|RGW) + rgw|RGW) echo "[CEPHADM] Deploying RGW on node $HOSTNAME" rgw ;; @@ -642,7 +665,6 @@ function configure_ceph { if is_ceph_enabled_for_service cinder; then POOLS+=($CINDER_CEPH_POOL) KEYS+=("client.$CINDER_CEPH_USER") - set_min_client_version fi if is_ceph_enabled_for_service c-bak; then @@ -659,8 +681,10 @@ function configure_ceph { [ "$ENABLE_CEPH_RGW" == "True" ] && SERVICES+=('rgw') enable_services - add_pools - create_keys + if [[ "$REMOTE_CEPH" = "False" ]]; then + add_pools + create_keys + fi client_config import_libvirt_secret_ceph @@ -677,13 +701,21 @@ function configure_ceph_manila { function cleanup_ceph { # Cleanup the service. - stop_ceph - delete_osd_dev + if [[ "$REMOTE_CEPH" == "False" ]]; then + stop_ceph + delete_osds + fi # purge ceph config file and keys $SUDO rm -f ${CEPH_CONF_DIR}/* if is_ceph_enabled_for_service nova; then - _undefine_virsh_secret + _undefine_virsh_secret + fi + if [[ "$CEPHADM_DEPLOY_NFS" != "True" ]]; then + stop_nfs_ganesha + cleanup_nfs_ganesha + cleanup_repo_nfs_ganesha fi + } function disable_cephadm { diff --git a/devstack/lib/common b/devstack/lib/common new file mode 100755 index 0000000..1b7b665 --- /dev/null +++ b/devstack/lib/common @@ -0,0 +1,149 @@ +#!/bin/bash + +# Allows driver to store NFS-Ganesha exports and export counter as +# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later, +# Ceph v12.2.2 or later, and OpenStack Queens or later. +MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True} +GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'} +# Remove "v" and "-stable" prefix/suffix tags +GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//") +if [[ "$CEPHADM_DEPLOY" = "True" ]]; then + FSNAME=${FSNAME:-'cephfs'} + CEPHFS_DATA_POOL="cephfs.$FSNAME.data" +else + CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data} +fi + +if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then + # default ganesha release based on ceph release + case $CEPH_RELEASE in + pacific) + GANESHA_RELEASE='3.5' + ;; + *) + GANESHA_RELEASE='5.0' + ;; + esac +fi + +# configure_repo_nfsganesha - Configure NFS Ganesha repositories +function configure_repo_nfsganesha { + if is_ubuntu; then + # NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major + # version; we can't use a build microversion unlike el8/el9 builds + case $GANESHA_RELEASE in + 3.*) + sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0 + sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0 + ;; + *) + GANESHA_PPA_VERSION="${GANESHA_RELEASE:0:1}" + sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-"$GANESHA_PPA_VERSION" + sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-"$GANESHA_PPA_VERSION" + ;; + esac + sudo apt-get -y update + elif is_fedora; then + local repo="" + case $GANESHA_RELEASE in + 3.*) + repo="centos-release-nfs-ganesha30" + ;; + *) + repo="centos-release-nfs-ganesha5" + ;; + esac + sudo dnf -y install ${repo} + fi +} + +function install_nfs_ganesha { + configure_repo_nfsganesha + NFS_GANESHA_PACKAGES="nfs-ganesha nfs-ganesha-ceph \ + nfs-ganesha-rados-urls nfs-ganesha-vfs" + if is_ubuntu; then + LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}" + NFS_GANESHA_PACKAGES="${LIBNTIRPC_PACKAGE} ${NFS_GANESHA_PACKAGES}" + fi + install_package $NFS_GANESHA_PACKAGES +} + +function configure_nfs_ganesha { + # Configure NFS-Ganesha to work with Manila's CephFS driver + rados_cmd="sudo rados -p ${CEPHFS_DATA_POOL}" + if [[ "$CEPHADM_DEPLOY" = "True" ]]; then + CEPHADM=${TARGET_BIN}/cephadm + rados_cmd="sudo $CEPHADM shell rados -p ${CEPHFS_DATA_POOL}" + fi + + + sudo mkdir -p /etc/ganesha/export.d + if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then + # Create an empty placeholder ganesha export index object + echo | $rados_cmd put ganesha-export-index - + cat </dev/null + RADOS_URLS { + ceph_conf = ${CEPH_CONF_FILE}; + userid = admin; +} + +CACHEINODE { + Dir_Max = 1; + Dir_Chunk = 0; + + Cache_FDs = false; + + NParts = 1; + Cache_Size = 1; +} + +EXPORT_DEFAULTS { + Attr_Expiration_Time = 0; +} + +%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index +EOF + else + sudo touch /etc/ganesha/export.d/INDEX.conf + echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf + fi +} + +function start_nfs_ganesha { + # NFS-Ganesha server cannot run alongwith with other kernel NFS server. + sudo systemctl stop nfs-server || true + sudo systemctl disable nfs-server || true + sudo systemctl enable nfs-ganesha + sudo systemctl start nfs-ganesha || ( + echo "Ganesha didn't start. Let's debug..." >&2 + sudo systemctl status nfs-ganesha || true + echo "**Ganesha conf file**" >&2 + sudo cat /etc/ganesha/ganesha.conf || true + echo "**Ganesha log file**" >&2 + sudo cat /var/log/ganesha/ganesha.log || true + echo "**Exiting**" >&2 + exit 1 + ) + echo "Standalone NFS-Ganesha started successfully!" >&2 +} + +function stop_nfs_ganesha { + sudo systemctl stop nfs-ganesha + sudo systemctl disable nfs-ganesha +} + +function cleanup_nfs_ganesha { + sudo systemctl stop nfs-ganesha + sudo systemctl disable nfs-ganesha + sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs +} + +# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories +# Usage: cleanup_repo_nfsganesha +function cleanup_repo_nfsganesha { + if is_ubuntu; then + sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*" + elif is_fedora; then + sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo + fi +} diff --git a/devstack/override-defaults b/devstack/override-defaults index 18afcd6..aa80ef1 100644 --- a/devstack/override-defaults +++ b/devstack/override-defaults @@ -20,3 +20,4 @@ if [[ $ENABLE_CEPH_CINDER == "True" ]]; then fi CEPHADM_DEPLOY=$(trueorfalse False CEPHADM_DEPLOY) +REMOTE_CEPH=$(trueorfalse False REMOTE_CEPH) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index c6063f1..b415f6e 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -40,10 +40,14 @@ elif [[ "$1" == "stack" && "$2" == "pre-install" ]]; then fi fi elif [[ "$1" == "stack" && "$2" == "install" ]]; then - if [[ "$CEPHADM_DEPLOY" = "True" ]]; then + if [[ "$CEPHADM_DEPLOY" = "True" && "$REMOTE_CEPH" = "False" ]]; then # Perform installation of service source echo_summary "[cephadm] Installing ceph" install_ceph + set_min_client_version + elif [[ "$CEPHADM_DEPLOY" = "True" && "$REMOTE_CEPH" = "True" ]]; then + echo "[CEPHADM] Remote Ceph: Skipping install" + get_cephadm else # FIXME(melwitt): This is a hack to get around a namespacing issue with # Paste and PasteDeploy. For stable/queens, we use the Pike UCA packages @@ -53,9 +57,8 @@ elif [[ "$1" == "stack" && "$2" == "install" ]]; then # newer version of it, while python-pastedeploy remains. The mismatch # between the install path of paste and paste.deploy causes Keystone to # fail to start, with "ImportError: cannot import name deploy." - if [[ "$TARGET_BRANCH" == stable/queens || "$TARGET_BRANCH" == master ]]; then - pip_install -U --force PasteDeploy - fi + pip_install -U --force PasteDeploy + install_package python-is-python3 fi elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then if [[ "$CEPHADM_DEPLOY" = "True" ]]; then diff --git a/devstack/settings b/devstack/settings index 01f74a7..ef9e4db 100644 --- a/devstack/settings +++ b/devstack/settings @@ -62,16 +62,16 @@ if (is_ceph_enabled_for_service manila); then MANILA_OPTGROUP_cephfsnfs1_cephfs_conf_path=${CEPH_CONF_FILE} MANILA_OPTGROUP_cephfsnfs1_cephfs_auth_id=${MANILA_CEPH_USER} MANILA_OPTGROUP_cephfsnfs1_cephfs_protocol_helper_type=NFS - MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP - MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE) - if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then - MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE} - MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL} - fi - - if [ "$CEPHADM_DEPLOY" = "True" ]; then + if [[ $CEPHADM_DEPLOY_NFS == "True" ]]; then MANILA_OPTGROUP_cephfsnfs1_cephfs_nfs_cluster_id=${FSNAME} + else + MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP + MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE) + if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then + MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE} + MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL} + fi fi fi fi diff --git a/tox.ini b/tox.ini index fe80eb7..271c0c2 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ commands = bash -c "find {toxinidir} \ [testenv:docs] deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} -r{toxinidir}/doc/requirements.txt commands = rm -rf doc/build