diff --git a/.gitreview b/.gitreview index e1bf63ba7a..c88fea6ca6 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/devstack.git +defaultbranch=unmaintained/yoga diff --git a/.zuul.yaml b/.zuul.yaml index 067d3f5b08..a1f63caae0 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,21 +1,3 @@ -- pragma: - # NOTE(gtema): this is required for the changes in SDK feature/r1 branch to - # be using devstack - # TODO(gtema): delete this once r1 branch is merged into master - implied-branches: - - master - - feature/r1 - -- nodeset: - name: openstack-single-node - nodes: - - name: controller - label: ubuntu-xenial - groups: - - name: tempest - nodes: - - controller - - nodeset: name: openstack-single-node-focal nodes: @@ -36,36 +18,6 @@ nodes: - controller -- nodeset: - name: openstack-single-node-xenial - nodes: - - name: controller - label: ubuntu-xenial - groups: - - name: tempest - nodes: - - controller - -- nodeset: - name: devstack-single-node-centos-7 - nodes: - - name: controller - label: centos-7 - groups: - - name: tempest - nodes: - - controller - -- nodeset: - name: devstack-single-node-centos-8-stream - nodes: - - name: controller - label: centos-8-stream - groups: - - name: tempest - nodes: - - controller - - nodeset: name: devstack-single-node-centos-9-stream nodes: @@ -76,26 +28,6 @@ nodes: - controller -- nodeset: - name: devstack-single-node-opensuse-15 - nodes: - - name: controller - label: opensuse-15 - groups: - - name: tempest - nodes: - - controller - -- nodeset: - name: devstack-single-node-fedora-latest - nodes: - - name: controller - label: fedora-35 - groups: - - name: tempest - nodes: - - controller - - nodeset: name: devstack-single-node-debian-bullseye nodes: @@ -106,76 +38,6 @@ nodes: - controller -- nodeset: - name: devstack-single-node-openeuler-20.03-sp2 - nodes: - - name: controller - label: openEuler-20-03-LTS-SP2 - groups: - - name: tempest - nodes: - - controller - -- nodeset: - name: openstack-two-node - nodes: - - name: controller - label: ubuntu-xenial - - name: compute1 - label: ubuntu-xenial - groups: - # Node where tests are executed and test results collected - - name: tempest - nodes: - - controller - # Nodes running the compute service - - name: compute - nodes: - - controller - - compute1 - # Nodes that are not the controller - - name: subnode - nodes: - - compute1 - # Switch node for multinode networking setup - - name: switch - nodes: - - controller - # Peer nodes for multinode networking setup - - name: peers - nodes: - - compute1 - -- nodeset: - name: openstack-two-node-centos-8-stream - nodes: - - name: controller - label: centos-8-stream - - name: compute1 - label: centos-8-stream - groups: - # Node where tests are executed and test results collected - - name: tempest - nodes: - - controller - # Nodes running the compute service - - name: compute - nodes: - - controller - - compute1 - # Nodes that are not the controller - - name: subnode - nodes: - - compute1 - # Switch node for multinode networking setup - - name: switch - nodes: - - controller - # Peer nodes for multinode networking setup - - name: peers - nodes: - - compute1 - - nodeset: name: openstack-two-node-focal nodes: @@ -236,36 +98,6 @@ nodes: - compute1 -- nodeset: - name: openstack-two-node-xenial - nodes: - - name: controller - label: ubuntu-xenial - - name: compute1 - label: ubuntu-xenial - groups: - # Node where tests are executed and test results collected - - name: tempest - nodes: - - controller - # Nodes running the compute service - - name: compute - nodes: - - controller - - compute1 - # Nodes that are not the controller - - name: subnode - nodes: - - compute1 - # Switch node for multinode networking setup - - name: switch - nodes: - - controller - # Peer nodes for multinode networking setup - - name: peers - nodes: - - compute1 - - nodeset: name: openstack-three-node-focal nodes: @@ -353,7 +185,6 @@ required-projects: - opendev.org/openstack/devstack roles: - - zuul: opendev.org/openstack/devstack-gate - zuul: opendev.org/openstack/openstack-zuul-jobs vars: devstack_localrc: @@ -658,30 +489,14 @@ description: | Simple multinode test to verify multinode functionality on devstack side. This is not meant to be used as a parent job. + vars: + devstack_localrc: + MYSQL_REDUCE_MEMORY: true # NOTE(ianw) Platform tests have traditionally been non-voting because # we often have to rush things through devstack to stabilise the gate, # and these platforms don't have the round-the-clock support to avoid # becoming blockers in that situation. -- job: - name: devstack-platform-centos-8-stream - parent: tempest-full-py3 - description: CentOS 8 Stream platform test - nodeset: devstack-single-node-centos-8-stream - voting: false - timeout: 9000 - vars: - configure_swap_size: 4096 - -- job: - name: devstack-platform-centos-9-stream - parent: tempest-full-py3 - description: CentOS 9 Stream platform test - nodeset: devstack-single-node-centos-9-stream - timeout: 9000 - vars: - configure_swap_size: 4096 - - job: name: devstack-platform-debian-bullseye parent: tempest-full-py3 @@ -723,20 +538,6 @@ # Enable Neutron ML2/OVS services q-agt: true -- job: - name: devstack-platform-openEuler-20.03-SP2 - parent: tempest-full-py3 - description: openEuler 20.03 SP2 platform test - nodeset: devstack-single-node-openeuler-20.03-sp2 - voting: false - timeout: 9000 - vars: - configure_swap_size: 4096 - devstack_localrc: - # NOTE(wxy): OVN package is not supported by openEuler yet. Build it - # from source instead. - OVN_BUILD_FROM_SOURCE: True - - job: name: devstack-no-tls-proxy parent: tempest-full-py3 @@ -748,30 +549,6 @@ devstack_services: tls-proxy: false -- job: - name: devstack-platform-fedora-latest - parent: tempest-full-py3 - description: Fedora latest platform test - nodeset: devstack-single-node-fedora-latest - voting: false - vars: - configure_swap_size: 4096 - # Python 3.10 dependency issues; see - # https://bugs.launchpad.net/horizon/+bug/1960204 - devstack_services: - horizon: false - -- job: - name: devstack-platform-fedora-latest-virt-preview - parent: tempest-full-py3 - description: Fedora latest platform test using the virt-preview repo. - nodeset: devstack-single-node-fedora-latest - voting: false - vars: - configure_swap_size: 4096 - devstack_localrc: - ENABLE_FEDORA_VIRT_PREVIEW_REPO: true - - job: name: devstack-tox-base parent: devstack @@ -845,17 +622,18 @@ - devstack - devstack-ipv6 - devstack-enforce-scope - - devstack-platform-fedora-latest - - devstack-platform-centos-8-stream - - devstack-platform-centos-9-stream - - devstack-platform-debian-bullseye - - devstack-platform-openEuler-20.03-SP2 + - devstack-platform-debian-bullseye: + voting: false - devstack-multinode - devstack-unit-tests - openstack-tox-bashate - - ironic-tempest-bios-ipmi-direct-tinyipa + - ironic-tempest-bios-ipmi-direct-tinyipa: + voting: false - swift-dsvm-functional - grenade: + # NOTE(elod.illes): needs to be set back as voting, when grenade + # patch merges: https://review.opendev.org/919818 + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -877,15 +655,12 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - openstacksdk-functional-devstack: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - tempest-ipv6-only: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - nova-ceph-multistore: + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -893,7 +668,6 @@ jobs: - devstack - devstack-ipv6 - - devstack-platform-centos-9-stream - devstack-enforce-scope - devstack-multinode - devstack-unit-tests @@ -906,13 +680,15 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - ironic-tempest-bios-ipmi-direct-tinyipa + # TODO(ykarel): ironic-tempest job is broken in this branch, + # so it is set as non-voting in check queue and commented out here + # in gate queue until the job gets fixed. + # - ironic-tempest-bios-ipmi-direct-tinyipa - swift-dsvm-functional - grenade: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - openstacksdk-functional-devstack: + # NOTE(elod.illes): needs to be set back as voting, when grenade + # patch merges: https://review.opendev.org/919818 + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -920,10 +696,13 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - nova-ceph-multistore: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + # TODO(elod.illes): nova-ceph-multistore is broken in this branch, + # so it is set as non-voting in check queue and commented out here + # in gate queue until the job gets fixed. + # - nova-ceph-multistore: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ # Please add a note on each job and conditions for the job not # being experimental any more, so we can keep this list somewhat # pruned. @@ -937,14 +716,8 @@ # * neutron-functional-with-uwsgi: maintained by neutron for functional # test. Next cycle we can remove this one if things turn out to be # stable engouh with uwsgi. - # * neutron-tempest-with-uwsgi: maintained by neutron for tempest test. - # Next cycle we can remove this if everything run out stable enough. # * nova-multi-cell: maintained by nova and currently non-voting in the # check queue for nova changes but relies on devstack configuration - # * devstack-platform-fedora-latest-virt-preview: Maintained by lyarwood - # for Nova to allow early testing of the latest versions of Libvirt and - # QEMU. Should only graduate out of experimental if it ever moves into - # the check queue for Nova. experimental: jobs: @@ -952,7 +725,6 @@ - nova-next - neutron-fullstack-with-uwsgi - neutron-functional-with-uwsgi - - neutron-tempest-with-uwsgi - devstack-plugin-ceph-tempest-py3: irrelevant-files: - ^.*\.rst$ @@ -973,8 +745,4 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - devstack-platform-fedora-latest-virt-preview - - devstack-no-tls-proxy - periodic: - jobs: - devstack-no-tls-proxy diff --git a/functions-common b/functions-common index b2cf9d99c6..20440920ec 100644 --- a/functions-common +++ b/functions-common @@ -646,8 +646,10 @@ function git_clone { echo "the project to the \$PROJECTS variable in the job definition." die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration" fi - # '--branch' can also take tags - git_timed clone $git_clone_flags $git_remote $git_dest --branch $git_ref + git_timed clone --no-checkout $git_clone_flags $git_remote $git_dest + cd $git_dest + git_timed fetch $git_clone_flags origin $git_ref + git_timed checkout FETCH_HEAD elif [[ "$RECLONE" = "True" ]]; then # if it does exist then simulate what clone does if asked to RECLONE cd $git_dest @@ -673,6 +675,18 @@ function git_clone { fi fi + # NOTE(ianw) 2022-04-13 : commit [1] has broken many assumptions + # about how we clone and work with repos. Mark them safe globally + # as a work-around. + # + # NOTE(danms): On bionic (and likely others) git-config may write + # ~stackuser/.gitconfig if not run with sudo -H. Using --system + # writes these changes to /etc/gitconfig which is more + # discoverable anyway. + # + # [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9 + sudo git config --system --add safe.directory ${git_dest} + # print out the results so we know what change was used in the logs cd $git_dest git show --oneline | head -1 @@ -1563,6 +1577,7 @@ function write_user_unit_file { local command="$2" local group=$3 local user=$4 + local env_vars="$5" local extra="" if [[ -n "$group" ]]; then extra="Group=$group" @@ -1576,6 +1591,9 @@ function write_user_unit_file { iniset -sudo $unitfile "Service" "KillMode" "process" iniset -sudo $unitfile "Service" "TimeoutStopSec" "300" iniset -sudo $unitfile "Service" "ExecReload" "$KILL_PATH -HUP \$MAINPID" + if [[ -n "$env_vars" ]] ; then + iniset -sudo $unitfile "Service" "Environment" "$env_vars" + fi if [[ -n "$group" ]]; then iniset -sudo $unitfile "Service" "Group" "$group" fi @@ -1590,6 +1608,7 @@ function write_uwsgi_user_unit_file { local command="$2" local group=$3 local user=$4 + local env_vars="$5" local unitfile="$SYSTEMD_DIR/$service" mkdir -p $SYSTEMD_DIR @@ -1604,6 +1623,9 @@ function write_uwsgi_user_unit_file { iniset -sudo $unitfile "Service" "NotifyAccess" "all" iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100" + if [[ -n "$env_vars" ]] ; then + iniset -sudo $unitfile "Service" "Environment" "$env_vars" + fi if [[ -n "$group" ]]; then iniset -sudo $unitfile "Service" "Group" "$group" fi @@ -1651,10 +1673,14 @@ function _run_under_systemd { local systemd_service="devstack@$service.service" local group=$3 local user=${4:-$STACK_USER} + if [[ -z "$user" ]]; then + user=$STACK_USER + fi + local env_vars="$5" if [[ "$command" =~ "uwsgi" ]] ; then - write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" + write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars" else - write_user_unit_file $systemd_service "$cmd" "$group" "$user" + write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars" fi $SYSTEMCTL enable $systemd_service @@ -1675,18 +1701,20 @@ function is_running { # If the command includes shell metachatacters (;<>*) it must be run using a shell # If an optional group is provided sg will be used to run the # command as that group. -# run_process service "command-line" [group] [user] +# run_process service "command-line" [group] [user] [env_vars] +# env_vars must be a space separated list of variable assigments, ie: "A=1 B=2" function run_process { local service=$1 local command="$2" local group=$3 local user=$4 + local env_vars="$5" local name=$service time_start "run_process" if is_service_enabled $service; then - _run_under_systemd "$name" "$command" "$group" "$user" + _run_under_systemd "$name" "$command" "$group" "$user" "$env_vars" fi time_stop "run_process" } diff --git a/lib/cinder b/lib/cinder index b029fa0db4..01c7f4f3dc 100644 --- a/lib/cinder +++ b/lib/cinder @@ -91,6 +91,8 @@ CINDER_VOLUME_CLEAR=$(echo ${CINDER_VOLUME_CLEAR} | tr '[:upper:]' '[:lower:]') # Default to lioadm CINDER_ISCSI_HELPER=${CINDER_ISCSI_HELPER:-lioadm} +VOLUME_TYPE_MULTIATTACH=${VOLUME_TYPE_MULTIATTACH:-multiattach} + # EL and SUSE should only use lioadm if is_fedora || is_suse; then if [[ ${CINDER_ISCSI_HELPER} != "lioadm" ]]; then @@ -552,8 +554,13 @@ function start_cinder { fi run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF" - run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF" - run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF" + # Tune glibc for Python Services using single malloc arena for all threads + # and disabling dynamic thresholds to reduce memory usage when using native + # threads directly or via eventlet.tpool + # https://www.gnu.org/software/libc/manual/html_node/Memory-Allocation-Tunables.html + malloc_tuning="MALLOC_ARENA_MAX=1 MALLOC_MMAP_THRESHOLD_=131072 MALLOC_TRIM_THRESHOLD_=262144" + run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF" "" "" "$malloc_tuning" + run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF" "" "" "$malloc_tuning" # NOTE(jdg): For cinder, startup order matters. To ensure that repor_capabilities is received # by the scheduler start the cinder-volume service last (or restart it) after the scheduler @@ -568,6 +575,23 @@ function stop_cinder { stop_process c-vol } +function create_one_type { + type_name=$1 + property_key=$2 + property_value=$3 + # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode + if is_service_enabled keystone; then + openstack --os-region-name="$REGION_NAME" volume type create --property $property_key="$property_value" $type_name + else + # TODO (e0ne): use openstack client once it will support cinder in noauth mode: + # https://bugs.launchpad.net/python-cinderclient/+bug/1755279 + local cinder_url + cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3 + OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create $type_name + OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key $type_name set $property_key="$property_value" + fi +} + # create_volume_types() - Create Cinder's configured volume types function create_volume_types { # Create volume types @@ -575,19 +599,13 @@ function create_volume_types { local be be_name for be in ${CINDER_ENABLED_BACKENDS//,/ }; do be_name=${be##*:} - # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode - if is_service_enabled keystone; then - openstack --os-region-name="$REGION_NAME" volume type create --property volume_backend_name="${be_name}" ${be_name} - else - # TODO (e0ne): use openstack client once it will support cinder in noauth mode: - # https://bugs.launchpad.net/python-cinderclient/+bug/1755279 - local cinder_url - cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3 - OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create ${be_name} - OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key ${be_name} set volume_backend_name=${be_name} - fi + create_one_type $be_name "volume_backend_name" $be_name done + if [[ $ENABLE_VOLUME_MULTIATTACH == "True" ]]; then + create_one_type $VOLUME_TYPE_MULTIATTACH $VOLUME_TYPE_MULTIATTACH " True" + fi + # Increase quota for the service project if glance is using cinder, # since it's likely to occasionally go above the default 10 in parallel # test execution. diff --git a/lib/databases/mysql b/lib/databases/mysql index 0f45273d4b..3a4fa940aa 100644 --- a/lib/databases/mysql +++ b/lib/databases/mysql @@ -150,6 +150,16 @@ function configure_database_mysql { iniset -sudo $my_conf mysqld log-queries-not-using-indexes 1 fi + if [[ "$MYSQL_REDUCE_MEMORY" == "True" ]]; then + iniset -sudo $my_conf mysqld read_buffer_size 64K + iniset -sudo $my_conf mysqld innodb_buffer_pool_size 16M + iniset -sudo $my_conf mysqld thread_stack 192K + iniset -sudo $my_conf mysqld thread_cache_size 8 + iniset -sudo $my_conf mysqld tmp_table_size 8M + iniset -sudo $my_conf mysqld sort_buffer_size 8M + iniset -sudo $my_conf mysqld max_allowed_packet 8M + fi + restart_service $MYSQL_SERVICE_NAME } diff --git a/lib/neutron-legacy b/lib/neutron-legacy index b906a1b2ff..016df43123 100644 --- a/lib/neutron-legacy +++ b/lib/neutron-legacy @@ -138,6 +138,9 @@ Q_NOTIFY_NOVA_PORT_DATA_CHANGES=${Q_NOTIFY_NOVA_PORT_DATA_CHANGES:-True} VIF_PLUGGING_IS_FATAL=${VIF_PLUGGING_IS_FATAL:-True} VIF_PLUGGING_TIMEOUT=${VIF_PLUGGING_TIMEOUT:-300} +# Allow to skip stopping of OVN services +SKIP_STOP_OVN=${SKIP_STOP_OVN:-False} + # The directory which contains files for Q_PLUGIN_EXTRA_CONF_FILES. # /etc/neutron is assumed by many of devstack plugins. Do not change. _Q_PLUGIN_EXTRA_CONF_PATH=/etc/neutron @@ -638,7 +641,7 @@ function stop_mutnauq { stop_mutnauq_other stop_mutnauq_l2_agent - if [[ $Q_AGENT == "ovn" ]]; then + if [[ $Q_AGENT == "ovn" && $SKIP_STOP_OVN != "True" ]]; then stop_ovn fi } @@ -931,6 +934,9 @@ function _configure_neutron_service { configure_keystone_authtoken_middleware $NEUTRON_CONF nova nova + # Configuration for placement client + configure_keystone_authtoken_middleware $NEUTRON_CONF placement placement + # Configure plugin neutron_plugin_configure_service } diff --git a/lib/neutron_plugins/ovn_agent b/lib/neutron_plugins/ovn_agent index 927896b70b..634ec1e328 100644 --- a/lib/neutron_plugins/ovn_agent +++ b/lib/neutron_plugins/ovn_agent @@ -384,6 +384,8 @@ function install_ovn { sudo mkdir -p $OVS_PREFIX/var/log/ovn sudo chown $(whoami) $OVS_PREFIX/var/log/ovn else + # Load fixup_ovn_centos + source ${TOP_DIR}/tools/fixup_stuff.sh fixup_ovn_centos install_package $(get_packages openvswitch) install_package $(get_packages ovn) @@ -575,6 +577,7 @@ function init_ovn { rm -f $OVS_DATADIR/.*.db.~lock~ sudo rm -f $OVN_DATADIR/*.db sudo rm -f $OVN_DATADIR/.*.db.~lock~ + sudo rm -f $OVN_RUNDIR/*.sock } function _start_ovs { @@ -601,7 +604,7 @@ function _start_ovs { dbcmd+=" --remote=db:hardware_vtep,Global,managers $OVS_DATADIR/vtep.db" fi dbcmd+=" $OVS_DATADIR/conf.db" - _run_process ovsdb-server "$dbcmd" + _run_process ovsdb-server "$dbcmd" "" "$STACK_GROUP" "root" # Note: ovn-controller will create and configure br-int once it is started. # So, no need to create it now because nothing depends on that bridge here. @@ -689,7 +692,7 @@ function start_ovn { local cmd="/bin/bash $SCRIPTDIR/ovn-ctl --no-monitor start_northd" local stop_cmd="/bin/bash $SCRIPTDIR/ovn-ctl stop_northd" - _run_process ovn-northd "$cmd" "$stop_cmd" + _run_process ovn-northd "$cmd" "$stop_cmd" "$STACK_GROUP" "root" else _start_process "$OVN_NORTHD_SERVICE" fi diff --git a/lib/neutron_plugins/ovs_source b/lib/neutron_plugins/ovs_source index 9ae5555afb..164d574c42 100644 --- a/lib/neutron_plugins/ovs_source +++ b/lib/neutron_plugins/ovs_source @@ -87,9 +87,15 @@ function prepare_for_ovs_compilation { install_package kernel-devel-$KERNEL_VERSION install_package kernel-headers-$KERNEL_VERSION + if is_service_enabled tls-proxy; then + install_package openssl-devel + fi elif is_ubuntu ; then install_package linux-headers-$KERNEL_VERSION + if is_service_enabled tls-proxy; then + install_package libssl-dev + fi fi } diff --git a/lib/tempest b/lib/tempest index 45046632b4..fb44b1b3ec 100644 --- a/lib/tempest +++ b/lib/tempest @@ -58,7 +58,7 @@ BUILD_TIMEOUT=${BUILD_TIMEOUT:-196} # This must be False on stable branches, as master tempest # deps do not match stable branch deps. Set this to True to # have tempest installed in DevStack by default. -INSTALL_TEMPEST=${INSTALL_TEMPEST:-"True"} +INSTALL_TEMPEST=${INSTALL_TEMPEST:-"False"} # This variable is passed directly to pip install inside the common tox venv # that is created @@ -71,6 +71,17 @@ TEMPEST_VOLUME_VENDOR=${TEMPEST_VOLUME_VENDOR:-$TEMPEST_DEFAULT_VOLUME_VENDOR} TEMPEST_DEFAULT_STORAGE_PROTOCOL="iSCSI" TEMPEST_STORAGE_PROTOCOL=${TEMPEST_STORAGE_PROTOCOL:-$TEMPEST_DEFAULT_STORAGE_PROTOCOL} +# Glance/Image variables +# When Glance image import is enabled, image creation is asynchronous and images +# may not yet be active when tempest looks for them. In that case, we poll +# Glance every TEMPEST_GLANCE_IMPORT_POLL_INTERVAL seconds for the number of +# times specified by TEMPEST_GLANCE_IMPORT_POLL_LIMIT. If you are importing +# multiple images, set TEMPEST_GLANCE_IMAGE_COUNT so the poller does not quit +# too early (though it will not exceed the polling limit). +TEMPEST_GLANCE_IMPORT_POLL_INTERVAL=${TEMPEST_GLANCE_IMPORT_POLL_INTERVAL:-1} +TEMPEST_GLANCE_IMPORT_POLL_LIMIT=${TEMPEST_GLANCE_IMPORT_POLL_LIMIT:-12} +TEMPEST_GLANCE_IMAGE_COUNT=${TEMPEST_GLANCE_IMAGE_COUNT:-1} + # Neutron/Network variables IPV6_ENABLED=$(trueorfalse True IPV6_ENABLED) IPV6_SUBNET_ATTRIBUTES_ENABLED=$(trueorfalse True IPV6_SUBNET_ATTRIBUTES_ENABLED) @@ -117,6 +128,13 @@ function set_tempest_venv_constraints { (cd $REQUIREMENTS_DIR && git show master:upper-constraints.txt 2>/dev/null || git show origin/master:upper-constraints.txt) > $tmp_c + # NOTE(gmann): we need to set the below env var pointing to master + # constraints even that is what default in tox.ini. Otherwise it can + # create the issue for grenade run where old and new devstack can have + # different tempest (old and master) to install. For detail problem, + # refer to the https://bugs.launchpad.net/devstack/+bug/2003993 + export UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/master + export TOX_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/master else echo "Using $TEMPEST_VENV_UPPER_CONSTRAINTS constraints in Tempest virtual env." cat $TEMPEST_VENV_UPPER_CONSTRAINTS > $tmp_c @@ -127,6 +145,48 @@ function set_tempest_venv_constraints { fi } +# Makes a call to glance to get a list of active images, ignoring +# ramdisk and kernel images. Takes 3 arguments, an array and two +# variables. The array will contain the list of active image UUIDs; +# if an image with ``DEFAULT_IMAGE_NAME`` is found, its UUID will be +# set as the value of *both* other parameters. +function get_active_images { + declare -n img_array=$1 + declare -n img_id=$2 + declare -n img_id_alt=$3 + + # start with a fresh array in case we are called multiple times + img_array=() + + while read -r IMAGE_NAME IMAGE_UUID; do + if [ "$IMAGE_NAME" = "$DEFAULT_IMAGE_NAME" ]; then + img_id="$IMAGE_UUID" + img_id_alt="$IMAGE_UUID" + fi + img_array+=($IMAGE_UUID) + done < <(openstack --os-cloud devstack-admin image list --property status=active | awk -F'|' '!/^(+--)|ID|aki|ari/ { print $3,$2 }') +} + +function poll_glance_images { + declare -n image_array=$1 + declare -n image_id=$2 + declare -n image_id_alt=$3 + local -i poll_count + + poll_count=$TEMPEST_GLANCE_IMPORT_POLL_LIMIT + while (( poll_count-- > 0 )) ; do + sleep $TEMPEST_GLANCE_IMPORT_POLL_INTERVAL + get_active_images image_array image_id image_id_alt + if (( ${#image_array[*]} >= $TEMPEST_GLANCE_IMAGE_COUNT )) ; then + return + fi + done + local msg + msg="Polling limit of $TEMPEST_GLANCE_IMPORT_POLL_LIMIT exceeded; " + msg+="poll interval was $TEMPEST_GLANCE_IMPORT_POLL_INTERVAL sec" + warn $LINENO "$msg" +} + # configure_tempest() - Set config files, create data dirs, etc function configure_tempest { if [[ "$INSTALL_TEMPEST" == "True" ]]; then @@ -168,13 +228,21 @@ function configure_tempest { declare -a images if is_service_enabled glance; then - while read -r IMAGE_NAME IMAGE_UUID; do - if [ "$IMAGE_NAME" = "$DEFAULT_IMAGE_NAME" ]; then - image_uuid="$IMAGE_UUID" - image_uuid_alt="$IMAGE_UUID" + get_active_images images image_uuid image_uuid_alt + + if (( ${#images[*]} < $TEMPEST_GLANCE_IMAGE_COUNT )); then + # Glance image import is asynchronous and may be configured + # to do image conversion. If image import is being used, + # it's possible that this code is being executed before the + # import has completed and there may be no active images yet. + if [[ "$GLANCE_USE_IMPORT_WORKFLOW" == "True" ]]; then + poll_glance_images images image_uuid image_uuid_alt + if (( ${#images[*]} < $TEMPEST_GLANCE_IMAGE_COUNT )); then + echo "Only found ${#images[*]} image(s), was looking for $TEMPEST_GLANCE_IMAGE_COUNT" + exit 1 + fi fi - images+=($IMAGE_UUID) - done < <(openstack --os-cloud devstack-admin image list --property status=active | awk -F'|' '!/^(+--)|ID|aki|ari/ { print $3,$2 }') + fi case "${#images[*]}" in 0) @@ -394,7 +462,7 @@ function configure_tempest { # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_compute_max_microversion" # for stable branch on each release which should be changed from "latest" to max supported version of that release. local tempest_compute_min_microversion=${TEMPEST_COMPUTE_MIN_MICROVERSION:-None} - local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"latest"} + local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"2.90"} # Reset microversions to None where v2.0 is running which does not support microversion. # Both "None" means no microversion testing. if [[ "$TEMPEST_COMPUTE_TYPE" == "compute_legacy" ]]; then @@ -487,7 +555,7 @@ function configure_tempest { fi iniset $TEMPEST_CONFIG volume-feature-enabled volume_revert $(trueorfalse False TEMPEST_VOLUME_REVERT_TO_SNAPSHOT) local tempest_volume_min_microversion=${TEMPEST_VOLUME_MIN_MICROVERSION:-None} - local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"latest"} + local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"3.68"} if [ "$tempest_volume_min_microversion" == "None" ]; then inicomment $TEMPEST_CONFIG volume min_microversion else @@ -536,13 +604,17 @@ function configure_tempest { iniset $TEMPEST_CONFIG volume storage_protocol "$TEMPEST_STORAGE_PROTOCOL" fi + if [[ $ENABLE_VOLUME_MULTIATTACH == "True" ]]; then + iniset $TEMPEST_CONFIG volume volume_type_multiattach $VOLUME_TYPE_MULTIATTACH + fi + # Placement Features # Set the microversion range for placement. # Setting [None, latest] range of microversion which allow Tempest to run all microversions tests. # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_placement_max_microversion" # for stable branch on each release which should be changed from "latest" to max supported version of that release. local tempest_placement_min_microversion=${TEMPEST_PLACEMENT_MIN_MICROVERSION:-None} - local tempest_placement_max_microversion=${TEMPEST_PLACEMENT_MAX_MICROVERSION:-"latest"} + local tempest_placement_max_microversion=${TEMPEST_PLACEMENT_MAX_MICROVERSION:-"1.39"} if [ "$tempest_placement_min_microversion" == "None" ]; then inicomment $TEMPEST_CONFIG placement min_microversion else @@ -613,6 +685,8 @@ function configure_tempest { iniset $TEMPEST_CONFIG auth admin_project_name '' fi iniset $TEMPEST_CONFIG enforce_scope glance "$GLANCE_ENFORCE_SCOPE" + # NOTE(gmann): system scope is supported for <=stable/yoga only + iniset $TEMPEST_CONFIG glance_api system_scope_supported "True" iniset $TEMPEST_CONFIG enforce_scope cinder "$CINDER_ENFORCE_SCOPE" @@ -629,13 +703,13 @@ function configure_tempest { local tmp_cfg_file tmp_cfg_file=$(mktemp) cd $TEMPEST_DIR - if [[ "$OFFLINE" != "True" ]]; then - tox -revenv-tempest --notest - fi local tmp_u_c_m tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX) set_tempest_venv_constraints $tmp_u_c_m + if [[ "$OFFLINE" != "True" ]]; then + tox -revenv-tempest --notest + fi tox -evenv-tempest -- pip install -c $tmp_u_c_m -r requirements.txt rm -f $tmp_u_c_m @@ -672,7 +746,28 @@ function configure_tempest { DISABLE_NETWORK_API_EXTENSIONS+=", l3_agent_scheduler" fi - local network_api_extensions=${NETWORK_API_EXTENSIONS:-"all"} + DEFAULT_NET_EXT="address-scope,agent,allowed-address-pairs,auto-allocated-topology" + DEFAULT_NET_EXT+=",availability_zone,binding,default-subnetpools,dhcp_agent_scheduler" + DEFAULT_NET_EXT+=",dvr,ext-gw-mode,external-net,extra_dhcp_opt,extraroute,flavors" + DEFAULT_NET_EXT+=",l3-flavors,l3-ha,l3_agent_scheduler,multi-provider,net-mtu" + DEFAULT_NET_EXT+=",network-ip-availability,network_availability_zone,pagination" + DEFAULT_NET_EXT+=",port-security,project-id,provider,quotas,quota_details,rbac-policies" + DEFAULT_NET_EXT+=",revision-if-match,router,router_availability_zone,security-group,service-type,sorting" + DEFAULT_NET_EXT+=",standard-attr-description,standard-attr-revisions,standard-attr-tag,standard-attr-timestamp" + DEFAULT_NET_EXT+=",subnet-service-types,subnet_allocation,net-mtu-writable,ip-substring-filtering" + DEFAULT_NET_EXT+=",availability_zone_filter,filter-validation,empty-string-filtering,port-mac-address-regenerate" + DEFAULT_NET_EXT+=",port-security-groups-filtering,fip-port-details,binding-extended" + DEFAULT_NET_EXT+=",subnet_onboard,l3-port-ip-change-not-allowed,agent-resources-synced" + DEFAULT_NET_EXT+=",floatingip-pools,rbac-security-groups,subnetpool-prefix-ops,router-admin-state-down-before-update" + DEFAULT_NET_EXT+=",rbac-subnetpool,tag-ports-during-bulk-creation,stateful-security-group,address-group,extraroute-atomic" + DEFAULT_NET_EXT+=",port-numa-affinity-policy,rbac-address-scope,security-groups-remote-address-group,trunk,trunk-details" + DEFAULT_NET_EXT+=",rbac-address-group,port-device-profile" + DEFAULT_NET_EXT+=",multiple-external-gateways,qos-pps-minimum,l3-ext-ndp-proxy,rbac-bgpvpn" + DEFAULT_NET_EXT+=",qos-pps,ecmp_routes,bgp,floating-ip-port-forwarding-port-ranges" + #New in Yoga + DEFAULT_NET_EXT+=",security-groups-shared-filtering,security-groups-normalized-cidr,quota-check-limit" + DEFAULT_NET_EXT+=",port-resource-request-groups" + local network_api_extensions=${NETWORK_API_EXTENSIONS:-$DEFAULT_NET_EXT} if [[ ! -z "$DISABLE_NETWORK_API_EXTENSIONS" ]]; then # Enabled extensions are either the ones explicitly specified or those available on the API endpoint network_api_extensions=${NETWORK_API_EXTENSIONS:-$(iniget $tmp_cfg_file network-feature-enabled api_extensions | tr -d " ")} @@ -684,7 +779,10 @@ function configure_tempest { fi iniset $TEMPEST_CONFIG network-feature-enabled api_extensions $network_api_extensions # Swift API Extensions - local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-"all"} + DEFAULT_SWIFT_OPT="account_quotas,bulk_delete,bulk_upload,container_quotas" + DEFAULT_SWIFT_OPT+=",container_sync,crossdomain,formpost,ratelimit,slo" + DEFAULT_SWIFT_OPT+=",staticweb,tempauth,tempurl,versioned_writes" + local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$DEFAULT_SWIFT_OPT} if [[ ! -z "$DISABLE_OBJECT_STORAGE_API_EXTENSIONS" ]]; then # Enabled extensions are either the ones explicitly specified or those available on the API endpoint object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$(iniget $tmp_cfg_file object-storage-feature-enabled discoverable_apis | tr -d " ")} @@ -693,7 +791,18 @@ function configure_tempest { fi iniset $TEMPEST_CONFIG object-storage-feature-enabled discoverable_apis $object_storage_api_extensions # Cinder API Extensions - local volume_api_extensions=${VOLUME_API_EXTENSIONS:-"all"} + DEFAULT_VOL_EXT="OS-SCH-HNT,backups,capabilities,cgsnapshots,consistencygroups" + DEFAULT_VOL_EXT+=",encryption,os-admin-actions,os-availability-zone" + DEFAULT_VOL_EXT+=",os-extended-services,os-extended-snapshot-attributes" + DEFAULT_VOL_EXT+=",os-hosts,os-quota-class-sets,os-quota-sets" + DEFAULT_VOL_EXT+=",os-services,os-snapshot-actions,os-snapshot-manage" + DEFAULT_VOL_EXT+=",os-snapshot-unmanage,os-types-extra-specs,os-types-manage" + DEFAULT_VOL_EXT+=",os-used-limits,os-vol-host-attr,os-vol-image-meta" + DEFAULT_VOL_EXT+=",os-vol-mig-status-attr,os-vol-tenant-attr,os-volume-actions" + DEFAULT_VOL_EXT+=",os-volume-encryption-metadata,os-volume-manage" + DEFAULT_VOL_EXT+=",os-volume-transfer,os-volume-type-access" + DEFAULT_VOL_EXT+=",os-volume-unmanage,qos-specs,scheduler-stats" + local volume_api_extensions=${VOLUME_API_EXTENSIONS:-$DEFAULT_VOL_EXT} if [[ ! -z "$DISABLE_VOLUME_API_EXTENSIONS" ]]; then # Enabled extensions are either the ones explicitly specified or those available on the API endpoint volume_api_extensions=${VOLUME_API_EXTENSIONS:-$(iniget $tmp_cfg_file volume-feature-enabled api_extensions | tr -d " ")} @@ -709,7 +818,12 @@ function configure_tempest { # install_tempest() - Collect source and prepare function install_tempest { git_clone $TEMPEST_REPO $TEMPEST_DIR $TEMPEST_BRANCH - pip_install 'tox!=2.8.0' + # NOTE(gmann): Pinning tox<4.0.0 for stable/zed and lower. Tox 4.0.0 + # released after zed was released and has some incompatible changes + # and it is ok not to fix the issues caused by tox 4.0.0 in stable + # beanches jobs. We can continue testing the stable/zed and lower + # branches with tox<4.0.0 + pip_install 'tox!=2.8.0,<4.0.0' pushd $TEMPEST_DIR # NOTE(gmann): checkout the TEMPEST_BRANCH in case TEMPEST_BRANCH # is tag name not master. git_clone would not checkout tag because diff --git a/playbooks/tox/pre.yaml b/playbooks/tox/pre.yaml index d7e4670a80..68d5254251 100644 --- a/playbooks/tox/pre.yaml +++ b/playbooks/tox/pre.yaml @@ -5,4 +5,10 @@ bindep_profile: test bindep_dir: "{{ zuul_work_dir }}" - test-setup - - ensure-tox + # NOTE(gmann): Pinning tox<4.0.0 for stable/zed and lower. Tox 4.0.0 + # released after zed was released and has some incompatible changes + # and it is ok not to fix the issues caused by tox 4.0.0 in stable + # beanches jobs. We can continue testing the stable/zed and lower + # branches with tox<4.0.0 + - role: ensure-tox + ensure_tox_version: "<4" diff --git a/playbooks/tox/run-both.yaml b/playbooks/tox/run-both.yaml index e4043d8231..0f8af920e4 100644 --- a/playbooks/tox/run-both.yaml +++ b/playbooks/tox/run-both.yaml @@ -6,6 +6,12 @@ bindep_profile: test bindep_dir: "{{ zuul_work_dir }}" - test-setup - - ensure-tox + # NOTE(gmann): Pinning tox<4.0.0 for stable/zed and lower. Tox 4.0.0 + # released after zed was released and has some incompatible changes + # and it is ok not to fix the issues caused by tox 4.0.0 in stable + # beanches jobs. We can continue testing the stable/zed and lower + # branches with tox<4.0.0 + - role: ensure-tox + ensure_tox_version: "<4" - get-devstack-os-environment - tox diff --git a/roles/setup-devstack-source-dirs/tasks/main.yaml b/roles/setup-devstack-source-dirs/tasks/main.yaml index 294c29cd29..cb7c6e3af8 100644 --- a/roles/setup-devstack-source-dirs/tasks/main.yaml +++ b/roles/setup-devstack-source-dirs/tasks/main.yaml @@ -43,9 +43,9 @@ base_branch={{ devstack_sources_branch }} if git branch -a | grep "$base_branch" > /dev/null ; then git checkout $base_branch - elif [[ "$base_branch" == stable/* ]]; then + elif [[ "$base_branch" == stable/* ]] || [[ "$base_branch" == unmaintained/* ]]; then # Look for an eol tag for the stable branch. - eol_tag=${base_branch#stable/}-eol + eol_tag="${base_branch#*/}-eol" if git tag -l |grep $eol_tag >/dev/null; then git checkout $eol_tag git reset --hard $eol_tag diff --git a/stack.sh b/stack.sh index 0082b99f11..443c1db1f8 100755 --- a/stack.sh +++ b/stack.sh @@ -316,7 +316,22 @@ function _install_rdo { sudo dnf -y install https://rdoproject.org/repos/openstack-${rdo_release}/rdo-release-${rdo_release}.el8.rpm fi elif [[ $DISTRO == "rhel9" ]]; then - sudo curl -L -o /etc/yum.repos.d/delorean-deps.repo http://trunk.rdoproject.org/centos9-master/delorean-deps.repo + install_package wget + # We need to download rdo-release package using wget as installing with dnf from repo.fedoraproject.org fails in + # FIPS enabled systems after https://bugzilla.redhat.com/show_bug.cgi?id=2157951 + # Until we can pull rdo-release from a server which supports EMS, this workaround is doing wget, which does + # not relies on openssl but on gnutls, and then install it locally using rpm + TEMPRDODIR=$(mktemp -d) + if [[ "$TARGET_BRANCH" == "master" ]]; then + # rdo-release.el9.rpm points to latest RDO release, use that for master + wget -P $TEMPRDODIR https://rdoproject.org/repos/rdo-release.el9.rpm + else + # For stable branches use corresponding release rpm + rdo_release=$(echo $TARGET_BRANCH | sed "s|stable/||g") + wget -P $TEMPRDODIR https://rdoproject.org/repos/openstack-${rdo_release}/rdo-release-${rdo_release}.el9.rpm + fi + sudo rpm -ivh $TEMPRDODIR/rdo-release*rpm + rm -rf $TEMPRDODIR fi sudo dnf -y update } diff --git a/stackrc b/stackrc index e48fd81d16..cc7cbfad6e 100644 --- a/stackrc +++ b/stackrc @@ -15,7 +15,9 @@ source $RC_DIR/functions # Set the target branch. This is used so that stable branching # does not need to update each repo below. -TARGET_BRANCH=master +# NOTE(minhnlh2): The yoga release is no longer supported, so we +# update the target branch from stable/yoga to unmaintained/yoga +TARGET_BRANCH=unmaintained/yoga # Cycle trailing projects need to branch later than the others. TRAILING_TARGET_BRANCH=master @@ -193,6 +195,11 @@ ADDITIONAL_VENV_PACKAGES=${ADITIONAL_VENV_PACKAGES:-""} # (currently only implemented for MySQL backend) DATABASE_QUERY_LOGGING=$(trueorfalse False DATABASE_QUERY_LOGGING) +# This can be used to reduce the amount of memory mysqld uses while running. +# These are unscientifically determined, and could reduce performance or +# cause other issues. +MYSQL_REDUCE_MEMORY=$(trueorfalse False MYSQL_REDUCE_MEMORY) + # Set a timeout for git operations. If git is still running when the # timeout expires, the command will be retried up to 3 times. This is # in the format for timeout(1); @@ -287,8 +294,8 @@ REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-$TARGET_BRANCH} # Tempest test suite TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git} -TEMPEST_BRANCH=${TEMPEST_BRANCH:-$BRANCHLESS_TARGET_BRANCH} -TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-master} +TEMPEST_BRANCH=${TEMPEST_BRANCH:-41.0.0 } +TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-$REQUIREMENTS_DIR/upper-constraints.txt} ############## diff --git a/tests/test_libs_from_pypi.sh b/tests/test_libs_from_pypi.sh index 839e3a1328..c30ba728ba 100755 --- a/tests/test_libs_from_pypi.sh +++ b/tests/test_libs_from_pypi.sh @@ -96,19 +96,7 @@ function test_libs_exist { echo "test_libs_exist PASSED" } -function test_branch_master { - for lib in $ALL_LIBS; do - if [[ ${GITBRANCH[$lib]} != "master" ]]; then - echo "GITBRANCH for $lib not master (${GITBRANCH[$lib]})" - exit 1 - fi - done - - echo "test_branch_master PASSED" -} - set -o errexit test_libs_exist -test_branch_master test_all_libs_upto_date diff --git a/tests/test_refs.sh b/tests/test_refs.sh index 0f9aa4a5ca..d9b328294a 100755 --- a/tests/test_refs.sh +++ b/tests/test_refs.sh @@ -15,7 +15,7 @@ echo "Ensuring we don't have crazy refs" -REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH'` +REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH' | grep -v 'TEMPEST_BRANCH'` rc=$? if [[ $rc -eq 0 ]]; then echo "Branch defaults must be one of the *TARGET_BRANCH values. Found:" diff --git a/tox.ini b/tox.ini index ec764abc87..37531ea80b 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,7 @@ commands = bash -c "find {toxinidir} \ [testenv:docs] deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/yoga} -r{toxinidir}/doc/requirements.txt allowlist_externals = bash setenv = diff --git a/unstack.sh b/unstack.sh index 4b57b6e344..a36af3fb59 100755 --- a/unstack.sh +++ b/unstack.sh @@ -181,3 +181,8 @@ fi clean_pyc_files rm -Rf $DEST/async + +# Clean any safe.directory items we wrote into the global +# gitconfig. We can identify the relevant ones by checking that they +# point to somewhere in our $DEST directory. +sudo sed -i "\+directory = ${DEST}+ d" /etc/gitconfig