From 5400050a7ee684915e678dbf4db05afb0d593a5a Mon Sep 17 00:00:00 2001 From: Michael Chapman Date: Fri, 15 Nov 2013 15:07:07 +1100 Subject: [PATCH 1/2] Change grizzly stable branch to install grizzly Grizzly branch was installing havana before. Oops. --- data/hiera_data/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hiera_data/common.yaml b/data/hiera_data/common.yaml index ae00dbd..39aad84 100644 --- a/data/hiera_data/common.yaml +++ b/data/hiera_data/common.yaml @@ -28,7 +28,7 @@ nova::compute::libvirt::vncserver_listen: 0.0.0.0 nova::compute::force_config_drive: true # information for package repos -coe::base::openstack_release: havana +coe::base::openstack_release: grizzly coe::base::package_repo: cloud_archive # used for testing From 0092fdf829b46f133caf290b8ac06867216b9751 Mon Sep 17 00:00:00 2001 From: Michael Chapman Date: Mon, 18 Nov 2013 13:38:44 +1100 Subject: [PATCH 2/2] Remove stack-builder Stack-builder tools have been broken out into their own repository and can now be downloaded via pip. This patch updates the README and removes the stack-builder tools. --- README.md | 43 +-- stack-builder/README.md | 224 ----------- stack-builder/bin/sb | 84 ---- stack-builder/build.py | 387 ------------------- stack-builder/clean.py | 82 ---- stack-builder/debug.py | 13 - stack-builder/fragment.py | 83 ---- stack-builder/fragments/SIGNAL_TEMPLATE | 1 - stack-builder/fragments/WAIT_TEMPLATE | 6 - stack-builder/fragments/apt-packages | 2 - stack-builder/fragments/bash-top | 1 - stack-builder/fragments/build-config.sh | 60 --- stack-builder/fragments/build-sig-stack | 2 - stack-builder/fragments/cloud-init-log | 2 - stack-builder/fragments/compute-wait-control | 4 - stack-builder/fragments/control-sig-compute | 1 - stack-builder/fragments/control-wait-compute | 38 -- stack-builder/fragments/copy-user-yaml | 2 - stack-builder/fragments/fqdn-fix | 7 - stack-builder/fragments/hiera-config | 12 - stack-builder/fragments/openstack-config.sh | 53 --- stack-builder/fragments/puppet-agent | 1 - stack-builder/fragments/puppet-master-fact | 3 - stack-builder/fragments/puppet-modules | 14 - stack-builder/fragments/puppet-modules-lite | 15 - stack-builder/fragments/puppet-setup | 2 - stack-builder/fragments/puppet-site | 4 - stack-builder/fragments/pw-mirror-network | 23 -- stack-builder/fragments/stack-wait-build | 4 - stack-builder/fragments/test-nova | 9 - stack-builder/hiera_config.py | 40 -- stack-builder/metadata.py | 67 ---- stack-builder/rebuild.py | 131 ------- 33 files changed, 5 insertions(+), 1415 deletions(-) delete mode 100644 stack-builder/README.md delete mode 100755 stack-builder/bin/sb delete mode 100644 stack-builder/build.py delete mode 100644 stack-builder/clean.py delete mode 100644 stack-builder/debug.py delete mode 100644 stack-builder/fragment.py delete mode 100644 stack-builder/fragments/SIGNAL_TEMPLATE delete mode 100644 stack-builder/fragments/WAIT_TEMPLATE delete mode 100644 stack-builder/fragments/apt-packages delete mode 100644 stack-builder/fragments/bash-top delete mode 100644 stack-builder/fragments/build-config.sh delete mode 100644 stack-builder/fragments/build-sig-stack delete mode 100644 stack-builder/fragments/cloud-init-log delete mode 100644 stack-builder/fragments/compute-wait-control delete mode 100644 stack-builder/fragments/control-sig-compute delete mode 100644 stack-builder/fragments/control-wait-compute delete mode 100644 stack-builder/fragments/copy-user-yaml delete mode 100644 stack-builder/fragments/fqdn-fix delete mode 100644 stack-builder/fragments/hiera-config delete mode 100644 stack-builder/fragments/openstack-config.sh delete mode 100644 stack-builder/fragments/puppet-agent delete mode 100644 stack-builder/fragments/puppet-master-fact delete mode 100644 stack-builder/fragments/puppet-modules delete mode 100644 stack-builder/fragments/puppet-modules-lite delete mode 100644 stack-builder/fragments/puppet-setup delete mode 100644 stack-builder/fragments/puppet-site delete mode 100644 stack-builder/fragments/pw-mirror-network delete mode 100644 stack-builder/fragments/stack-wait-build delete mode 100644 stack-builder/fragments/test-nova delete mode 100644 stack-builder/hiera_config.py delete mode 100644 stack-builder/metadata.py delete mode 100644 stack-builder/rebuild.py diff --git a/README.md b/README.md index 1b3887b..76048e9 100644 --- a/README.md +++ b/README.md @@ -84,44 +84,11 @@ and run through the 'Deploy Your First VM' section of this document: ## Spinning up virtual machines with Openstack -(Experimental) - -The python scripts under stack-builder can be used to instantiate scenarios on an openstack cluster. To do this, clone this repository, add stackbuilder/bin to your PATH, and add stackbuilder to your PYTHONPATH. It is not necessary to install the modules or librarian to your local machine when running in this manner, but the openstack clients and the python yaml library are needed. - - curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.10.1.tar.gz - tar xvfz virtualenv-1.10.1.tar.gz - cd virtualenv-1.10.1 - python virtualenv.py test - cd test - source bin/activate - pip install python-novaclient==2.14.1 - pip install python-quantumclient==2.2.3 - pip install python-keystoneclient==0.3.2 - pip install PyYaml - git clone "https://github.com/CiscoSystems/openstack-installer" - cd openstack-installer - export PATH=`pwd`/stack-builder/bin:$PATH - export PYTHONPATH=`pwd`/stack-builder:$PYTHONPATH - -The scripts are currently limited by a quantum bug that means admin credentials are required to launch. Using standard user credentials will result in networks that have no dhcp agent scheduled to them. - - source openrc - -To create a basic 2 role cluster with a build, compute and control node outside of the jenkins environment, some configuration must be set either in data/heira\_data/user.yaml or by setting environment variables prefixed with "jenkins\_" - - export osi_user_internal_ip='%{ipaddress_eth1}' - export osi_user_tunnel_ip='%{ipaddress_eth1}' - export osi_conf_initial_ntp=ntp.esl.cisco.com - export osi_conf_installer_repo=CiscoSystems - export osi_conf_installer_branch=master - export osi_conf_build_server_domain_name=domain.name - export osi_conf_operatingsystem=Ubuntu - - sb make - - e824830b269544d39a632d89e0a1902c - -More information about this tool can be found under the stack-builder directory. +The data model in this repository can be consumed by the scenariobuilder tool. To install it, use pip: + + pip install scenariobuilder + +The 'sb' tool can then be used with Openstack credentials to instantiate the data model in VMs on an Openstack cloud. For more information see: https://github.com/CiscoSystems/scenariobuilder # Basic install against already provisioned nodes (Ubuntu 12.04.3 LTS): diff --git a/stack-builder/README.md b/stack-builder/README.md deleted file mode 100644 index 663c9ef..0000000 --- a/stack-builder/README.md +++ /dev/null @@ -1,224 +0,0 @@ -# stack-builder - -## Overview - -Stack-builder is a set of scripts that are intended to be used with openstack-installer. Given openstack credentials, stack-builder will create the appropriate resources on the given cloud to test a scenario described by openstack-installer, and will install openstack using the specified scenario. - -## Installation - -These instructions are copied almost directly from the jenkins job: - - curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.10.1.tar.gz - tar xvfz virtualenv-1.10.1.tar.gz - cd virtualenv-1.10.1 - python virtualenv.py test - cd test - source bin/activate - pip install python-novaclient==2.14.1 - pip install python-quantumclient==2.2.3 - pip install PyYaml - - source /home/jenkins-slave/installer_credentials/openrc - - git clone "https://github.com/CiscoSystems/openstack-installer" - - cd openstack-installer - - export PATH=`pwd`/stack-builder/bin:$PATH - export PYTHONPATH=`pwd`/stack-builder:$PYTHONPATH - - export osi_user_internal_ip='%{ipaddress_eth1}' - export osi_user_tunnel_ip='%{ipaddress_eth1}' - - export osi_conf_initial_ntp=ntp.esl.cisco.com - export osi_conf_installer_repo=CiscoSystems - export osi_conf_installer_branch=master - export osi_conf_operatingsystem=Ubuntu - export osi_conf_build_server_domain_name=domain.name - -## Commands - -For all commands, the data path refers to the data directory in openstack-installer, and defaults to './data', while the fragment path refers to the location of the bash snippets used to composed cloud-init scripts, and defaults to './stack-builder/fragments'. Current the only supported image is ubuntu and defaults to 'precise-x86\_64'. - -### Make - - usage: sb make [-h] [-i IMAGE] [-c CI_SUBNET_INDEX] [-s SCENARIO] [-p DATA_PATH] [-f FRAGMENT_PATH] [-d] - optional arguments: - -h, --help show this help message and exit - -i IMAGE, --image IMAGE - name of image to use - -s SCENARIO, --scenario SCENARIO - Scenario to run - -p DATA_PATH, --data_path DATA_PATH - Path to the data directory containing yaml config - -f FRAGMENT_PATH, --fragment_path FRAGMENT_PATH - Path to config fragments - -d, --debug print debug output - -Creates a cluster based on the given scenario. - -example: - - >sb make - 18dbb77039754173b68d9deabbb88505 - -### Get - - usage: sb get [-h] [-t TEST_ID] - - optional arguments: - -h, --help show this help message and exit - -t TEST_ID, --test_id TEST_ID - id of the test to get - -Gets info on all running clusters, or if -t is used, gets info on a single cluster. - -example: - - >sb get -t 18dbb77039754173b68d9deabbb88505 - Test ID: 18dbb77039754173b68d9deabbb88505 - 9d8112ab compute-server02 10.123.0.24 - c9873ca9 control-server 10.123.0.23 - 97c17070 cache 10.123.0.22 - b8eb5fde build-server 10.123.0.2 - -### Kill - - usage: sb kill [-h] [-t TEST_ID] - - optional arguments: - -h, --help show this help message and exit - -t TEST_ID, --test_id TEST_ID - id of the test to kill - -Destroys resources created by the test tool, except the shared resources: the external router and the ci network. If -t is used it will only destroy resources associated with the specified test id. - - >sb kill -t 18dbb77039754173b68d9deabbb88505 - Deleting instance 9d8112ab-1312-427c-b9e8-9f9231c9cc0e from test 18dbb77039754173b68d9deabbb88505 - Deleting instance c9873ca9-ce70-4159-968c-9cadac3b322c from test 18dbb77039754173b68d9deabbb88505 - Deleting instance 97c17070-bec2-46b2-8fa4-11354fda28a3 from test 18dbb77039754173b68d9deabbb88505 - Deleting instance b8eb5fde-1c19-4c2e-aec8-0d446165a1cb from test 18dbb77039754173b68d9deabbb88505 - deleted network ci-internal-18dbb77039754173b68d9deabbb88505 - deleted network ci-external-18dbb77039754173b68d9deabbb88505 - -### Frag - - usage: sb frag [-h] [-f FRAGMENT_DIR] [-y YAML_DIR] [-s SCENARIO] node - - positional arguments: - node node to build a fragment for - - optional arguments: - -h, --help show this help message and exit - -f FRAGMENT_DIR, --fragment_dir FRAGMENT_DIR - name of image to use - -y YAML_DIR, --yaml_dir YAML_DIR - name of image to use - -s SCENARIO, --scenario SCENARIO - Scenario to use - -Prints out the script that cloud-init will run on the specified node after fragments have been composed and substitutions have been made. This won't include substitutions that require run-time data such as build\_server\_ip. - -Example: - - sb frag build-server - - sb frag build-server - >#!/bin/bash - > ... - > [long series of commands] - -### Meta - - usage: sb meta [-h] [-y YAML_DIR] [-s SCENARIO] [-c CONFIG] node - - positional arguments: - node node to build metadata for - - optional arguments: - -h, --help show this help message and exit - -y YAML_DIR, --yaml_dir YAML_DIR - name of image to use - -s SCENARIO, --scenario SCENARIO - Scenario to use - -c CONFIG, --config CONFIG - Type of config to build - user or config - -Prints the yaml file that will be sent to the specified node. Can be either user or conf, for user.yaml and config.yaml respectively. - -Example: - - >sb frag build-server - [contents of config.yaml] - - >sb frag build-server -c user - [contents of user.yaml] - -### Wait - - usage: sb wait [-h] [-t TEST_ID] - - optional arguments: - -h, --help show this help message and exit - -t TEST_ID, --test_id TEST_ID - id of the build to wait for - -Waits for all VMs, or all VMs in the specified build, to signal their deployment is complete. This command requires a route to the ci network that the VMs are on. It's used by jenkins to detemine when it's ok to start running tests (tempest or otherwise) - -Example: - - sb wait -t 18dbb77039754173b68d9deabbb88505 - -### Log - - usage: sb log [-h] [-t TEST_ID] [-s SCENARIO] [-p DATA_PATH] - - optional arguments: - -h, --help show this help message and exit - -t TEST_ID, --test_id TEST_ID - id of the test to get logs from - -s SCENARIO, --scenario SCENARIO - Scenario of test - -p DATA_PATH, --data_path DATA_PATH - Path to the data directory containing yaml config - -Waits for deploy status to be complete, then copies logs from all servers, or from servers in the specified test. This command requires a route to the ci network that the VMs are on. It's used by Jenkins in liu of centralised logging and will likely go away at some point. - -Example: - - sb log -t 18dbb77039754173b68d9deabbb88505 - - -## Data Flow - -There are broadly two data categories of concern: data that controls the build environment, such as how many virtual machines to build and which networks to put them on, and data that controls the openstack cluster that is build using puppet. Stack-builder defines the former as conf data and the latter as user data. - -### User data - -Since user data is really just serving as an input to puppet, it config is quite simple to deal with: there are four yaml files in data/hiera\_data - user.yaml, jenkins.yaml, user.common and user.[scenario].yaml that are loaded into a single dictionary in python, along with any environment variables with the prefix osi\_user\_. The order or precedence is env. vars > user.yaml > jenkins.yaml > user.common > user.[scenario].yaml. After these have all been loaded and the dictionary has been created, this is written to a string and sent to every virtual machine as /root/user.yaml. This is then copied to /etc/puppet/data/hiera\_data/user.yaml by the deploy script. Because user.yaml has the highest precedence in hiera, this can be used to override any settings in the openstack puppet modules. - -The sb meta command can be used with the flag '-c user' to inspect the data that will be loaded. - -### Conf data + build configuration - -Config data is slightly more complex, since it effectively deals with preparing the cluster to run puppet. First, depending on which scenario has been selected, the script will load data/nodes/[scenario].yaml. This will define which VMs to create and which networks to create. The script will use the quantum API to create the appropriate networks along with ports for each VM. In the network section of the yaml file, some of the networks are also mappings themselves, such as build-server: networks: ci: cobbler\_node\_ip. A dictionary of runtime-config is created, with each of these mappings set as a key, and the value set as the IP address obtained for the appropriate port from quantum. So we end up with a dictionary that might look like this in a 2\_role scenario: - - runtime\_config = {'cobbler\_node\_ip' : '10.123.0.22', - 'control_node_ip' : '10.123.0.23'} - -This is the runtime config, and any data set here will override data from either environment variables or config files. - -Environment variables with the prefix osi\_conf\_ are put into the dictionary along with everything from data/config.yaml. Environment variables take precedence over config.yaml. - -This leaves us with a dictionary of all the config values we care about. - -This is written to a string and pushed to each node as /root/config.yaml - -The sb meta command can be used with the flag '-c conf' to inspect what will be loaded. - -Now that networks have been created and runtime+static config has been loaded, data/nodes/[scenario].yaml is examined again, this time to look at which fragments are specified for each node. Fragments are small snippets of bash in the stack-builder/fragments directory, and the list for each node will determine which ones are needed for each node. For example, the puppet master will need to run puppet apply, while the puppet agents will need to run puppet agent. Templating is in use in these fragments, and after they have been composed into a single string for each node, substitution occurs for each value in the runtime-config. This lets the agent nodes know where the master is, and lets compute nodes know where the controller is (from runtime data); lets nodes know which domain they are a part of, and which ntp server to sync with (needed to run puppet, from config.yaml), and any user specified overrides to change per run, such as a particular repository or branch to use for a puppet module (from environment variables). After templating has been handled, the final scripts are written to /root/deploy on each node. Cloud init will blindly execute whatever it finds there on each node. - -The sb frag command can be used to inspect what these build scripts will look like on each node - -One of the fragments will load /root/config.yaml and export all its values as environment variables with the prefix FACTER\_, making them available to the initial puppet setup. Be very careful when changing config.yaml, since for example, changing 'Ubuntu' to 'ubuntu' will prevent puppet from running, and in general system facts can be overridden causing strange behavior. - diff --git a/stack-builder/bin/sb b/stack-builder/bin/sb deleted file mode 100755 index 5d7d263..0000000 --- a/stack-builder/bin/sb +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -from build import make, cli_get, log, wait -from rebuild import rebuild -from clean import kill -from fragment import show as frag_show -from metadata import show as meta_show - -from novaclient.v1_1 import client as nclient -from quantumclient.quantum import client as qclient -from keystoneclient.v2_0 import client as kclient - -import argparse -import os - -def main(): - user = os.environ.get('OS_USERNAME') - tenant = os.environ.get('OS_TENANT_NAME') - auth_url = os.environ.get('OS_AUTH_URL') - password = os.environ.get('OS_PASSWORD') - n = nclient.Client(user, password, tenant, auth_url, service_type="compute", insecure=True) - q = qclient.Client('2.0', auth_url=auth_url, username=user, tenant_name=tenant, password=password) - k = kclient.Client(username=user, password=password, tenant_name=tenant, auth_url=auth_url) - - parser = argparse.ArgumentParser(description='Stack Builder: a tool for deploying Openstack on Openstack using puppet') - subparsers = parser.add_subparsers() - - parser_make = subparsers.add_parser('make', help='Make a test run') - parser_make.add_argument('-i', '--image', default='precise-x86_64', help='name of image to use') - parser_make.add_argument('-s', '--scenario', default='2_role', help='Scenario to run') - parser_make.add_argument('-p', '--data_path', default='./data', help='Path to the data directory containing yaml config') - parser_make.add_argument('-f', '--fragment_path', default='./stack-builder/fragments', help='Path to config fragments') - parser_make.add_argument('-d', '--debug', action='store_true', help='print debug output') - parser_make.add_argument('-n', '--public_network', default='public', help='The name of the public network on the cluster') - parser_make.add_argument('-a', '--nameserver', default='171.70.168.183', help='The DNS nameserver to use for all nodes being booted.') - parser_make.set_defaults(func=make) - - parser_rebuild = subparsers.add_parser('rebuild', help='Rebuild a test using existing resources') - parser_rebuild.add_argument('test_id', help='id of the test to get') - parser_rebuild.add_argument('-i', '--image', default='precise-x86_64', help='name of image to use') - parser_rebuild.add_argument('-s', '--scenario', default='2_role', help='Scenario to run') - parser_rebuild.add_argument('-p', '--data_path', default='./data', help='Path to the data directory containing yaml config') - parser_rebuild.add_argument('-f', '--fragment_path', default='./stack-builder/fragments', help='Path to config fragments') - parser_rebuild.add_argument('-d', '--debug', action='store_true', help='print debug output') - parser_rebuild.add_argument('-n', '--public_network', default='public', help='The name of the public network on the cluster') - parser_rebuild.set_defaults(func=rebuild) - - parser_get = subparsers.add_parser('get', help='Get info on current test runs') - parser_get.add_argument('-t', '--test_id', default=None, help='id of the test to get') - parser_get.set_defaults(func=cli_get) - - parser_kill = subparsers.add_parser('kill', help='Destroy resources associated with test runs') - parser_kill.add_argument('-t', '--test_id', default=None, help='id of the test to kill') - parser_kill.set_defaults(func=kill) - - parser_frag = subparsers.add_parser('frag', help='Print deploy scripts that will be generated') - parser_frag.add_argument('node', help='node to build a fragment for') - parser_frag.add_argument('-f', '--fragment_dir', default='./stack-builder/fragments', help='name of image to use') - parser_frag.add_argument('-y', '--yaml_dir', default='./data', help='name of image to use') - parser_frag.add_argument('-s', '--scenario', default='2_role', help='Scenario to use') - parser_frag.set_defaults(func=frag_show) - - parser_log = subparsers.add_parser('log', help='Get logs from test nodes') - parser_log.add_argument('-t', '--test_id', default=None, help='id of the test to get logs from') - parser_log.add_argument('-s', '--scenario', default='2_role', help='Scenario of test') - parser_log.add_argument('-p', '--data_path', default='./data', help='Path to the data directory containing yaml config') - parser_log.set_defaults(func=log) - - parser_wait = subparsers.add_parser('wait', help='Wait for all test nodes to complete building') - parser_wait.add_argument('-t', '--test_id', default=None, help='id of the build to wait for') - parser_wait.set_defaults(func=wait) - - parser_meta = subparsers.add_parser('meta', help='Print the metadata dictionary for a test run') - parser_meta.add_argument('node', help='node to build metadata for') - parser_meta.add_argument('-y', '--yaml_dir', default='./data', help='name of image to use') - parser_meta.add_argument('-s', '--scenario', default='2_role', help='Scenario to use') - parser_meta.add_argument('-c', '--config', default='config', help='Type of config to build - user or config') - parser_meta.set_defaults(func=meta_show) - - args = parser.parse_args() - - args.func(n, q, k, args) - -if __name__ == '__main__': - main() diff --git a/stack-builder/build.py b/stack-builder/build.py deleted file mode 100644 index d2e1a10..0000000 --- a/stack-builder/build.py +++ /dev/null @@ -1,387 +0,0 @@ -#!/usr/bin/env python -""" - stack-builder.build - ~~~~~~~~~~~~~~~~~~~ - - This module is reponsible for creating VMs on the target cloud, - and pushing the appropriate metadata and init scripts to them - -""" -import debug -import subprocess -import os -import uuid -import quantumclient -import fragment -import yaml -import json -import time -import urllib2 - -from metadata import build_metadata -from debug import dprint - -cloud_init="""#cloud-config -runcmd: - - [chmod, ug+x, /root/deploy] - - [sh, /root/deploy] - - echo 'complete' > /var/www/deploy - -output: {all: '| tee -a /var/log/cloud-init-output.log'} -""" - -def build_server_deploy(): - with open ('./stack-builder/fragments/build-config.sh', 'r') as b: - return b.read() - -def build_server_hiera_config(): - with open('./stack-builder/hiera_config.py', 'r') as b: - return b.read() - -def stack_server_deploy(build_node_ip): - with open ('./stack-builder/fragments/openstack-config.sh', 'r') as b: - return b.read().replace('%build_server_ip%', build_node_ip) - -def build_nic_net_list(networks): - return [{'net-id': network['id'], 'port-id': '', 'v4-fixed-ip': ''} for network in networks] - -def build_nic_port_list(ports): - return [{'net-id': '', 'port-id': port, 'v4-fixed-ip': ''} for port in ports] - -def make_network(q, ci_network_name, index=0): - networks = q.list_networks() - - if ci_network_name not in [network['name'] for network in networks['networks']]: - dprint("q.create_network({'network': {'name':" + ci_network_name + ", 'admin_state_up': True}})['network']") - test_net = q.create_network({'network': {'name': ci_network_name, 'admin_state_up': True}})['network'] - - else: - for net in networks['networks']: - if net['name'] == ci_network_name: - test_net = net - return test_net - -def make_subnet(q, ci_network_name, test_net, index=1, dhcp=True, gateway=False, dns_nameserver="171.70.168.183"): - subnets = q.list_subnets() - - if ci_network_name not in [subnet['name'] for subnet in subnets['subnets']]: - dprint("CI subnet " + str(index) + " doesn't exist. Creating ...") - try: - # internal networks - if not gateway: - dprint("create_subnet({'subnet': { 'name':" + ci_network_name + str(index) + ",\n" + - " 'network_id': " + test_net['id'] + ",\n" + - " 'ip_version': 4,\n" + - " 'cidr': '10.2." + str(index) + ".0/24',\n" + - " 'enable_dhcp': True,\n" + - " 'gateway_ip': None}})['subnet']\n") - - test_subnet = q.create_subnet({'subnet': { 'name': ci_network_name + str(index), - 'network_id': test_net['id'], - 'ip_version': 4, - 'cidr': '10.2.'+ str(index) + '.0/24', - 'enable_dhcp': dhcp, - 'gateway_ip': None - }})['subnet'] - - # the external network - else: - if ci_network_name == 'ci': - dprint("create_subnet({'subnet': { 'name':" + ci_network_name + ",\n" + - " 'network_id': " + test_net['id'] + ",\n" + - " 'ip_version': 4,\n" + - " 'cidr': '10." + str(index) + ".0.0/16',\n" + - " 'enable_dhcp':" + str(dhcp) + ",\n" + - " 'gateway_ip': '10." + str(index) + ".0.1'\n" + - " 'dns_nameservers: ['" + dns_nameserver + "']\n") - - test_subnet = q.create_subnet({'subnet': { 'name': ci_network_name, - 'network_id': test_net['id'], - 'ip_version': 4, - 'cidr': '10.' + str(index) + '.0.0/16', - 'enable_dhcp': dhcp, - #'gateway_ip' : '10.' + str(index) + '.0.1', - 'dns_nameservers': [unicode(dns_nameserver)] - }})['subnet'] - - except quantumclient.common.exceptions.QuantumClientException: - print "Couldn't create subnet!" - else: - for net in subnets['subnets']: - if net['name'] == ci_network_name: - test_subnet = net - return test_subnet - -def boot_puppetised_instance(n, name, image_name, nic_list, key='test2', os_flavor=u'm1.medium',deploy="",files=None, meta={}): - images = n.images.list() - for i,image in enumerate([image.name for image in images]): - if image == image_name: - boot_image = images[i] - - flavors = n.flavors.list() - for i,flavor in enumerate([flavor.name for flavor in flavors]): - if flavor == os_flavor: - boot_flavor = flavors[i] - - dprint("Booting " + name) - dprint("Boot image: " + str(boot_image)) - dprint("Boot flavor: " + str(boot_flavor)) - dprint("Boot nics: " + str(nic_list)) - dprint("Boot key: " + str(key)) - dprint("Boot deploy: " + str(deploy)) - dprint("Boot files: " + str(files)) - dprint("Boot meta: " + str(meta)) - - return n.servers.create(name, image=boot_image, flavor=boot_flavor, userdata=deploy, files=files, key_name=key, nics=nic_list, meta=meta) - -# Cisco internal network -def get_external_network(q): - for network in q.list_networks()['networks']: - if network['name'] == 'external': - return network - -# Used only for setting router gateway, VMs -# belong on external network -def get_public_network(q, public_network): - for network in q.list_networks()['networks']: - if network['name'] == unicode(public_network): - return network - -def get_external_router(q): - for router in q.list_routers()['routers']: - if router['name'] == 'ci': - return router - -def get_tenant_id(network): - return str(network['tenant_id']) - -# This can easily be problematic with multiple tenants, -# So make sure we get the one for the current tenant -def get_ci_network(q, n): - for network in q.list_networks()['networks']: - if network['name'] == 'ci' and network['tenant_id'] == get_tenant_id(n): - return network - -def get_ci_subnet(q, n): - for subnet in q.list_subnets()['subnets']: - if subnet['name'] == 'ci' and subnet['tenant_id'] == get_tenant_id(n): - return subnet - -def set_external_routing(q, subnet, public_network): - routers = q.list_routers() - ci_router = [router for router in routers['routers'] if router['name'] == 'ci'] - if len(ci_router) == 0: - ci_router = q.create_router( { 'router': { 'name': 'ci', - 'admin_state_up': 'true'} })['router'] - q.add_gateway_router(ci_router['id'], {'network_id': get_public_network(q, public_network)['id']}) - q.add_interface_router(ci_router['id'], {'subnet_id': subnet['id']}) - else: - ci_router = ci_router[0] - -def allocate_ports(q, network_id, test_id="", count=1): - # Bulk port creation - request_body = { "ports": [] } - for port in range(count): - request_body['ports'].append({ "network_id" : network_id, "name" : "ci-" + str(port) + '-' + test_id }) - return q.create_port(request_body)['ports'] - -def metadata_update(scenario_yaml, ports): - # IP addresses of particular nodes mapped to specified config - # values to go into hiera + build scripts. See data/nodes/2_role.yaml - # all values will also be mapped to a generic key: - # meta['hostname_networkname'] = ip - meta_update = {} - for node, props in scenario_yaml['nodes'].items(): - for network, mappings in props['networks'].items(): - if mappings != None: - for mapping in mappings: - meta_update[mapping] = str(ports[node][network][0]['fixed_ips'][0]['ip_address']) - # replace dashes since bash variables can't contain dashes - # but be careful when using this since hostnames can't contain underscores - # so they need to be converted back - meta_update[network + '_' + node.replace('-', '_')] = str(ports[node][network][0]['fixed_ips'][0]['ip_address']) - return meta_update - -# Not used atm -def make_key(n, test_id): - command = 'ssh-keygen -t rsa -q -N "" -f keys/'+test_id - process = subprocess.Popen(command) - n.keypairs.create(test_id, 'keys/'+test_id) - -def make(n, q, k, args): - image = args.image - ci_subnet_index = 123 # TODO fix inital setup stuff - scenario = args.scenario - data_path = args.data_path - fragment_path = args.fragment_path - public_network = args.public_network - nameserver = args.nameserver - - if args.debug: - debug.debug = True - - test_id = uuid.uuid4().hex - print test_id - - networks = {} - subnets = {} - ports = {} - - # Ci network with external route - # There can be only one of these per tenant - # because overlapping subnets + router doesn't work - networks['ci'] = make_network(q, 'ci') - subnets['ci'] = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True, dns_nameserver=nameserver) - set_external_routing(q, get_ci_subnet(q, networks['ci']), public_network) - ci_subnet_index = ci_subnet_index + 1 - - with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: - scenario_yaml = yaml.load(scenario_yaml_file.read()) - - # Find needed internal networks - for node, props in scenario_yaml['nodes'].items(): - for network in props['networks']: - if network != 'ci': # build network with NAT services - networks[network] = False - - - # Create internal networks - for network, gate in networks.items(): - if network != 'ci': - networks[network] = make_network(q, 'ci-' + network + '-' + test_id) - subnets[network] = make_subnet(q, 'ci-' + network + '-' + test_id, - networks[network], index=ci_subnet_index, gateway=gate) - ci_subnet_index = ci_subnet_index + 1 - - # There seems to be a bug in quantum where networks are not scheduled a dhcp agent unless a VM - # boots on that network without a pre-made port. So we boot an instance that will do this - # on all our networks - dummynets = [network for network in networks.values()] - dummy = boot_puppetised_instance(n, - 'dummy', - image, - build_nic_net_list(dummynets), - deploy=cloud_init, - meta={'ci_test_id' : test_id}, - os_flavor=u'm1.small' - ) - while dummy.status != u'ACTIVE': - dummy = n.servers.get(dummy) - dprint('dummy status: ' + str(dummy.status)) - dummy.delete() - - # Allocate ports - for node, props in scenario_yaml['nodes'].items(): - for network in props['networks']: - if node not in ports: - ports[node] = {} - ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) - else: - ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) - - dprint("networks") - for net, value in networks.items(): - dprint (net + str(value)) - dprint ("subnets") - for snet, value in subnets.items(): - dprint (snet + str(value)) - dprint ("ports") - dprint (json.dumps(ports,sort_keys=True, indent=4)) - - # config is a dictionary updated from env vars and user supplied - # yaml files to serve as input to hiera and build scripts - initial_config_meta = build_metadata(data_path, scenario, 'config') - hiera_config_meta = build_metadata(data_path, scenario, 'user') - global_config_meta = build_metadata(data_path, scenario, 'global') - - meta_update = metadata_update(scenario_yaml, ports) - - hiera_config_meta.update(meta_update) - initial_config_meta.update(meta_update) - - # fragment composition - deploy_files = {} - for node, props in scenario_yaml['nodes'].items(): - deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) - dprint(node + 'deploy:\n' + deploy_files[node]) - - user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) - initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) - global_config_yaml = yaml.dump(global_config_meta, default_flow_style=False) - - dprint('Config Yaml: \n' + str(initial_config_yaml)) - dprint('User Yaml: \n' + str(user_config_yaml)) - dprint('Global Yaml: \n' + str(global_config_yaml)) - - port_list = {} - for node, props in scenario_yaml['nodes'].items(): - nics = [] - for network in props['networks']: - nics.append(ports[node][network][0]['id']) - port_list[node] = build_nic_port_list(nics) - - for node, props in scenario_yaml['nodes'].items(): - boot_puppetised_instance(n, - node, - image, - port_list[node], - deploy=cloud_init, - files={ - u'/root/deploy' : deploy_files[node], - u'/root/user.yaml' : user_config_yaml, - u'/root/config.yaml' : initial_config_yaml, - u'/root/global.yaml' : global_config_yaml}, - meta={'ci_test_id' : test_id} - ) - -def cli_get(n,q,k,args): - run_instances = get(n,q,k,args) - for test_id, servers in run_instances.items(): - print "Test ID: " + test_id - for server in servers: - print "%-8.8s %16.16s %12.12s" % (server.id, server.name, str(server.networks['ci'][0])) - -def get(n, q, k, args): - run_instances = {} - instances = n.servers.list() - for instance in instances: - if 'ci_test_id' in instance.metadata: - if ((args.test_id and instance.metadata['ci_test_id'] == unicode(args.test_id)) or not args.test_id): - if instance.metadata['ci_test_id'] not in run_instances: - run_instances[instance.metadata['ci_test_id']] = [instance] - else: - run_instances[instance.metadata['ci_test_id']].append(instance) - return run_instances - -# Wait for deployment to finish -def wait(n, q, k, args): - test_id = args.test_id - - servers = get(n,q,k,args) - for server in servers[test_id]: - response = False - while not response: - try: - response = urllib2.urlopen('http://' + str(server.networks['ci'][0]) + '/deploy') - response = True - except: - time.sleep(15) - -# Get cloud-init logs -# TODO get all service logs -def log(n, q, k, args): - test_id = args.test_id - path = args.data_path - scenario = args.scenario - - servers = get(n,q,k,args) - for server in servers[test_id]: - response = False - while not response: - try: - response = urllib2.urlopen('http://' + str(server.networks['ci'][0]) + '/cloud-init-output.log') - with open('./' + str(server.name) + '-cloud-init.log', 'w') as output: - output.write(response.read()) - response = True - except: - time.sleep(20) diff --git a/stack-builder/clean.py b/stack-builder/clean.py deleted file mode 100644 index c241c1f..0000000 --- a/stack-builder/clean.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python -""" - stack-builder.clean - ~~~~~~~~~~~~~~~~~~~ - - This module is used by the kill subcommand - of the sb binary to destroy any virtual - resources created by an sb make command, with - the exception of the ci network, subnet and router - used as the deployment network and providing NAT. - -""" -from novaclient.v1_1 import client as nclient -from quantumclient.quantum import client as qclient -from time import sleep -import os - -def kill(n, q, k, args): - """ - Destroy either all virtual test resources, - or the resources from a particular run. - """ - test_id = None - if args.test_id: - test_id = args.test_id - - instances = n.servers.list() - - for instance in instances: - if 'ci_test_id' in instance.metadata: - if ((test_id and instance.metadata['ci_test_id'] == test_id) or not test_id): - print "Deleting instance " + instance.id + " from test " + instance.metadata['ci_test_id'] - n.servers.delete(instance) - - sleep(3) - - nets = q.list_networks() - subnets = q.list_subnets() - routers = q.list_routers() - ports = q.list_ports() - - for p in ports['ports']: - if p['name'][0:3] == 'ci-': - if ((test_id and p['name'][-32:] == test_id) or not test_id): - try: - q.delete_port(p['id']) - print 'deleted port ' + p['id'] - except: - pass - - for r in routers['routers']: - if r['name'][0:3] == 'ci-': - if ((test_id and r['name'][-32:] == test_id) or not test_id): - try: - q.remove_gateway_router(r['id']) - for subnet in subnets['subnets']: - if subnet['name'] == r['name']: - q.remove_interface_router(r['id'], { 'subnet_id' : subnet['id'] }) - - q.delete_router(r['id']) - print 'deleted router' + r['name'] - except: - pass - - for net in subnets['subnets']: - if net['name'][0:3] == 'ci-': - if ((test_id and net['name'][-32:] == test_id) or not test_id): - try: - q.delete_subnet(net['id']) - print 'deleted subnet ' + net['name'] - except: - pass - - for net in nets['networks']: - if net['name'][0:3] == 'ci-': - if ((test_id and net['name'][-32:] == test_id) or not test_id): - try: - q.delete_network(net['id']) - print 'deleted network ' + net['name'] - except: - pass - diff --git a/stack-builder/debug.py b/stack-builder/debug.py deleted file mode 100644 index 8f0079a..0000000 --- a/stack-builder/debug.py +++ /dev/null @@ -1,13 +0,0 @@ -debug = False -even = True - -def dprint(string): - global even - global debug - if debug: - if even: - print '\x1b[34m' + str(string) + '\x1b[0m' - even = False - else: - print '\x1b[1;34m' + str(string) + '\x1b[0m' - even = True diff --git a/stack-builder/fragment.py b/stack-builder/fragment.py deleted file mode 100644 index 191ec17..0000000 --- a/stack-builder/fragment.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -import string -import yaml - -from metadata import build_metadata - -def signal_type(fragment): - if fragment[:5] == 'WAIT_': - return 'wait' - if fragment[:7] == 'SIGNAL_': - return 'signal' - return None - -class PercentTemplate(string.Template): - delimiter='%' - -def available_fragments(d='./stack-builder/fragments'): - return [d + '/' + f for f in os.listdir(d)] - -# Create a deploy script from a list of fragments -def build_deploy(fragment_dir, frag_list, metadata): - deploy_script = "" - for f in frag_list: - sig = signal_type(f) - if sig == 'wait': - # Wait syntax is as follows: - # WAIT_signal hostname1 hostnameN - with open(fragment_dir + '/WAIT_TEMPLATE', 'r') as temp: - spl = f.split(' ') - nodes = "" - for node in spl[1:]: - # Handle debug case where there is none of this metadata - if 'ci_'+node.replace('-', '_') not in metadata: - metadata['ci_'+node.replace('-', '_')] = "{"+node+"_ip}" - print "Fragment creation: Node " + node + " IP not present in metadata: using " + "{" + node + "_ip}" - nodes = nodes + metadata['ci_'+node.replace('-', '_')] + " " - - repl = { 'nodes': nodes, 'signal': spl[0].split('_')[1] } - frag = PercentTemplate(temp.read()).safe_substitute(repl) - deploy_script = deploy_script + frag + '\n' - - elif sig == 'signal': - with open(fragment_dir + '/SIGNAL_TEMPLATE', 'r') as temp: - spl = f.split('_') - frag = PercentTemplate(temp.read()).safe_substitute({'signal': spl[1]}) - deploy_script = deploy_script + frag + '\n' - else: - with open(fragment_dir + '/' + f, 'r') as frag: - deploy_script = deploy_script + frag.read() + '\n' - - return deploy_script - -def load_yaml_config(node_name, yaml_dir='./data', fragment_dir='./stack-builder/fragments', scenario='2_role'): - with open(yaml_dir+'/nodes/'+scenario+'.yaml', 'r') as yaml_file: - y = yaml.load(yaml_file.read()) - - if node_name not in y['nodes']: - print "No node listed for node " + node_name + " in scenario " + scenario - return None - if 'fragments' not in y['nodes'][node_name]: - print "No fragments listed for node " + node_name + " in scenario " + scenario - return None - - available = available_fragments(fragment_dir) - for fragment in y['nodes'][node_name]['fragments']: - if fragment_dir + '/' + fragment not in available and not signal_type(fragment): - print "Fragment '" + fragment + "' specified in scenario " + scenario + "does not exist " - - return [f for f in y['nodes'][node_name]['fragments']] - - -def compose(hostname, yaml_dir, fragment_dir, scenario, replacements): - fragments = load_yaml_config(hostname, yaml_dir, fragment_dir, scenario) - script = build_deploy(fragment_dir, fragments, replacements) - return PercentTemplate(script).safe_substitute(replacements) - -def show(n, q, k, args): - hostname = args.node - yaml_dir = args.yaml_dir - fragment_dir = args.fragment_dir - scenario = args.scenario - - print compose(hostname, yaml_dir, fragment_dir, scenario, build_metadata('./data', '2_role', 'config')) diff --git a/stack-builder/fragments/SIGNAL_TEMPLATE b/stack-builder/fragments/SIGNAL_TEMPLATE deleted file mode 100644 index 09968cf..0000000 --- a/stack-builder/fragments/SIGNAL_TEMPLATE +++ /dev/null @@ -1 +0,0 @@ -echo %{signal} >> /var/www/status diff --git a/stack-builder/fragments/WAIT_TEMPLATE b/stack-builder/fragments/WAIT_TEMPLATE deleted file mode 100644 index 9f04e64..0000000 --- a/stack-builder/fragments/WAIT_TEMPLATE +++ /dev/null @@ -1,6 +0,0 @@ -for node in %{nodes}; do - until [ $(curl http://$node/status | grep %{signal}) ]; do - echo "waited for $node" >> /root/waiting - sleep 1 - done -done diff --git a/stack-builder/fragments/apt-packages b/stack-builder/fragments/apt-packages deleted file mode 100644 index e23c750..0000000 --- a/stack-builder/fragments/apt-packages +++ /dev/null @@ -1,2 +0,0 @@ -apt-get update -apt-get install -y puppet git rubygems curl python-yaml apache2 diff --git a/stack-builder/fragments/bash-top b/stack-builder/fragments/bash-top deleted file mode 100644 index a9bf588..0000000 --- a/stack-builder/fragments/bash-top +++ /dev/null @@ -1 +0,0 @@ -#!/bin/bash diff --git a/stack-builder/fragments/build-config.sh b/stack-builder/fragments/build-config.sh deleted file mode 100644 index 431d558..0000000 --- a/stack-builder/fragments/build-config.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -#Password is "test" -usermod --password p1fhTXKKhbc0M root - -# Rstarmer's precise mirror -sed -i 's/archive.ubuntu.com/172.29.74.100/g' /etc/apt/sources.list - -# Naively bring up whatever ethernet devices seem to be attached -for i in 1 2 3 4 -do - ifconfig -a | grep eth$i - if [ $? -eq 0 ]; - then - ifconfig eth$i up - dhclient eth$i - fi -done - -# mount config drive to populate hiera -# overrides from metadata -mkdir -p /mnt/config -mount /dev/disk/by-label/config-2 /mnt/config - -apt-get update -apt-get install -y puppet git rubygems python-yaml - -# Facter fqdn will come from DNS unless we do this -echo "127.0.1.1 $(hostname).domain.name $(hostname)" >> /etc/hosts - -# Install puppet librarian and all COI modules -git clone https://github.com/michaeltchapman/openstack-installer.git /root/openstack-installer -cd /root/openstack-installer -mkdir -p vendor -export GEM_HOME=`pwd`/vendor -gem install thor --no-ri --no-rdoc -git clone https://github.com/bodepd/librarian-puppet-simple.git vendor/librarian-puppet-simple -export PATH=`pwd`/vendor/librarian-puppet-simple/bin/:$PATH -export git_protocol='https' -librarian-puppet install --verbose --path /etc/puppet/modules - -# TODO unhardcode this from setup. it's ugly. -sed -i "s/'192.168.242.100'/\"\$\{\:\:ipaddress_eth0\}\"/g" /root/openstack-installer/manifests/setup.pp - -cp -r /root/openstack-installer/data /etc/puppet -cp -r /root/openstack-installer/manifests /etc/puppet - -# Override hiera values that have been passed in -# through metadata -python /root/hiera_config.py - -# Install the latest puppet and purge the old puppet -puppet apply manifests/setup.pp - -# Install build server -puppet apply manifests/site.pp - -puppet plugin download --server build-server.domain.name - -# Notify other nodes that the puppet-master is ready -echo "up" > /var/www/status diff --git a/stack-builder/fragments/build-sig-stack b/stack-builder/fragments/build-sig-stack deleted file mode 100644 index d27af52..0000000 --- a/stack-builder/fragments/build-sig-stack +++ /dev/null @@ -1,2 +0,0 @@ -# Notify other nodes that the puppet-master is ready -echo "up" > /var/www/status diff --git a/stack-builder/fragments/cloud-init-log b/stack-builder/fragments/cloud-init-log deleted file mode 100644 index 896a87f..0000000 --- a/stack-builder/fragments/cloud-init-log +++ /dev/null @@ -1,2 +0,0 @@ -chmod a+r /var/log/cloud-init-output.log -ln -s /var/log/cloud-init-output.log /var/www/cloud-init-output.log diff --git a/stack-builder/fragments/compute-wait-control b/stack-builder/fragments/compute-wait-control deleted file mode 100644 index df2a53c..0000000 --- a/stack-builder/fragments/compute-wait-control +++ /dev/null @@ -1,4 +0,0 @@ -until [ $(curl http://%{controller_public_address}/status | grep up) ]; do - echo "waited for control" >> /root/waiting - sleep 1 -done diff --git a/stack-builder/fragments/control-sig-compute b/stack-builder/fragments/control-sig-compute deleted file mode 100644 index 98d530d..0000000 --- a/stack-builder/fragments/control-sig-compute +++ /dev/null @@ -1 +0,0 @@ -echo "up" >> /var/www/status diff --git a/stack-builder/fragments/control-wait-compute b/stack-builder/fragments/control-wait-compute deleted file mode 100644 index a1846f1..0000000 --- a/stack-builder/fragments/control-wait-compute +++ /dev/null @@ -1,38 +0,0 @@ -function ovs_node_count() { - e=$(ovs-vsctl show | grep remote_ip | wc -l) - return $e -} - -ready=false -ovs_node_count -node_count=$? - -# Wait until at least one node is available -until [ "$node_count" -gt 0 ]; do - sleep(1) - ovs_node_count - node_count=$? -done - -until [ $ready = 'true' ]; do - ovs_node_count - initnodes=$? - ready=true - - # Check status of each node that is connected to ovs - for node in $(ovs-vsctl show | grep remote_ip | cut -d '"' -f 2); do - if [ ! $(curl http://"$nodes"/status | grep up) ]; then - ready=false - echo node "not ready for test to run" - fi - done - sleep 5 - - ovs_node_count - newnodes=$? - - # if more nodes have come online then re-do the check - if [ "$initnodes" -lt "$newnodes" ]; then - ready=false - fi -done diff --git a/stack-builder/fragments/copy-user-yaml b/stack-builder/fragments/copy-user-yaml deleted file mode 100644 index 0319e60..0000000 --- a/stack-builder/fragments/copy-user-yaml +++ /dev/null @@ -1,2 +0,0 @@ -cp /root/user.yaml /etc/puppet/data/hiera_data -cp /root/config.yaml /etc/puppet/data diff --git a/stack-builder/fragments/fqdn-fix b/stack-builder/fragments/fqdn-fix deleted file mode 100644 index 483476e..0000000 --- a/stack-builder/fragments/fqdn-fix +++ /dev/null @@ -1,7 +0,0 @@ -# Fix fqdn so puppet apply doesn't give us one on the wrong domain -sed -i 's/\#supersede/supersede/g' /etc/dhcp/dhclient.conf; -sed -i 's/fugue.com home.vix.com/%{domain}/g' /etc/dhcp/dhclient.conf; -sed -i 's/domain-name,//g' /etc/dhcp/dhclient.conf -dhclient -r eth0 && dhclient eth0; - -ntpdate %{initial_ntp} diff --git a/stack-builder/fragments/hiera-config b/stack-builder/fragments/hiera-config deleted file mode 100644 index 43f94c2..0000000 --- a/stack-builder/fragments/hiera-config +++ /dev/null @@ -1,12 +0,0 @@ -# Set facts for facter from config.yaml -python /root/openstack-installer/stack-builder/hiera_config.py - -# Can't set bash env vars from shild processes -for i in `cat /root/fact_exports`; do export $i; done - -cp /root/config.yaml /etc/puppet/data -cp /root/user.yaml /etc/puppet/data/hiera_data -cp /root/global.yaml /etc/puppet/data/global_hiera_params/user.yaml -chmod a+r /etc/puppet/data/config.yaml -chmod a+r /etc/puppet/data/hiera_data/user.yaml -chmod a+r /etc/puppet/data/global_hiera_params/user.yaml diff --git a/stack-builder/fragments/openstack-config.sh b/stack-builder/fragments/openstack-config.sh deleted file mode 100644 index 008a956..0000000 --- a/stack-builder/fragments/openstack-config.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -usermod --password p1fhTXKKhbc0M root - -# Rstarmer's precise mirror -sed -i 's/archive.ubuntu.com/172.29.74.100/g' /etc/apt/sources.list - -for i in 0 1 2 3 -do - ifconfig -a | grep eth$i - if [ $? -eq 0 ]; - then - ifconfig eth$i up - dhclient eth$i -v - fi -done - - -# Facter fqdn will come from DNS unless we do this -echo "127.0.1.1 $(hostname).domain.name $(hostname)" >> /etc/hosts - -#mount config drive -mkdir -p /mnt/config -mount /dev/disk/by-label/config-2 /mnt/config - -apt-get update -apt-get install -y puppet git rubygems curl python-yaml - -git clone https://github.com/michaeltchapman/openstack-installer.git /root/openstack-installer -cd /root/openstack-installer -mkdir -p vendor -export GEM_HOME=`pwd`/vendor -gem install thor --no-ri --no-rdoc -git clone https://github.com/bodepd/librarian-puppet-simple.git vendor/librarian-puppet-simple -export PATH=`pwd`/vendor/librarian-puppet-simple/bin/:$PATH -export git_protocol='https' -librarian-puppet install --verbose --path /etc/puppet/modules - -sed -i 's/192.168.242.100/%build_server_ip%/g' /root/openstack-installer/manifests/setup.pp - -cp -r /root/openstack-installer/data /etc/puppet -cp -r /root/openstack-installer/manifests /etc/puppet - -puppet apply manifests/setup.pp - -until [ $(curl http://%build_server_ip%/status | grep up) ]; do - echo "waited for build" >> /root/waiting - sleep 1 -done - -puppet agent -t --server build-server.domain.name - - diff --git a/stack-builder/fragments/puppet-agent b/stack-builder/fragments/puppet-agent deleted file mode 100644 index 2cdeff4..0000000 --- a/stack-builder/fragments/puppet-agent +++ /dev/null @@ -1 +0,0 @@ -puppet agent -t --server build-server.domain.name || [ $? -eq 2 ] diff --git a/stack-builder/fragments/puppet-master-fact b/stack-builder/fragments/puppet-master-fact deleted file mode 100644 index 557028b..0000000 --- a/stack-builder/fragments/puppet-master-fact +++ /dev/null @@ -1,3 +0,0 @@ -# By default, setup.pp will omit some components -# required by puppet master -export FACTER_puppet_run_mode=master diff --git a/stack-builder/fragments/puppet-modules b/stack-builder/fragments/puppet-modules deleted file mode 100644 index fda9a34..0000000 --- a/stack-builder/fragments/puppet-modules +++ /dev/null @@ -1,14 +0,0 @@ -git clone https://github.com/%{installer_repo}/openstack-installer.git /root/openstack-installer -cd /root/openstack-installer -git checkout %{installer_branch} -mkdir -p vendor -export GEM_HOME=`pwd`/vendor -gem install thor --no-ri --no-rdoc -git clone https://github.com/bodepd/librarian-puppet-simple.git vendor/librarian-puppet-simple -export PATH=`pwd`/vendor/librarian-puppet-simple/bin/:$PATH -export git_protocol='https' -export openstack_version=%{openstack_version} -librarian-puppet install --verbose --path /etc/puppet/modules - -cp -r /root/openstack-installer/data /etc/puppet -cp -r /root/openstack-installer/manifests /etc/puppet diff --git a/stack-builder/fragments/puppet-modules-lite b/stack-builder/fragments/puppet-modules-lite deleted file mode 100644 index 40b7ecf..0000000 --- a/stack-builder/fragments/puppet-modules-lite +++ /dev/null @@ -1,15 +0,0 @@ -git clone https://github.com/%{installer_repo}/openstack-installer.git /root/openstack-installer -cd /root/openstack-installer -git checkout %{installer_branch} - -git clone https://github.com/stephenrjohnson/puppetlabs-puppet /etc/puppet/modules/puppet -git clone https://github.com/CiscoSystems/puppet-apt /etc/puppet/modules/apt -git clone https://github.com/puppetlabs/puppetlabs-inifile /etc/puppet/modules/inifile -git clone https://github.com/puppetlabs/puppetlabs-apache /etc/puppet/modules/apache -git clone https://github.com/puppetlabs/puppetlabs-stdlib /etc/puppet/modules/stdlib -git clone https://github.com/bodepd/scenario_node_terminus /etc/puppet/modules/scenario_node_terminus - -export FACTER_puppet_run_mode=agent - -cp -r /root/openstack-installer/data /etc/puppet -cp -r /root/openstack-installer/manifests /etc/puppet diff --git a/stack-builder/fragments/puppet-setup b/stack-builder/fragments/puppet-setup deleted file mode 100644 index 28bacbd..0000000 --- a/stack-builder/fragments/puppet-setup +++ /dev/null @@ -1,2 +0,0 @@ -# Install the latest puppet and purge the old puppet -puppet apply manifests/setup.pp diff --git a/stack-builder/fragments/puppet-site b/stack-builder/fragments/puppet-site deleted file mode 100644 index ba05fd8..0000000 --- a/stack-builder/fragments/puppet-site +++ /dev/null @@ -1,4 +0,0 @@ -# Install build server -puppet apply manifests/site.pp - -puppet plugin download --server build-server.domain.name diff --git a/stack-builder/fragments/pw-mirror-network b/stack-builder/fragments/pw-mirror-network deleted file mode 100644 index 8b352c4..0000000 --- a/stack-builder/fragments/pw-mirror-network +++ /dev/null @@ -1,23 +0,0 @@ -usermod --password p1fhTXKKhbc0M root - -# Rstarmer's precise mirror -# Different ubuntu images have different defaults -sed -i 's/nova.clouds.archive.ubuntu.com/%{apt_mirror_ip}/g' /etc/apt/sources.list -sed -i 's/archive.ubuntu.com/%{apt_mirror_ip}/g' /etc/apt/sources.list - -for i in 1 2 3 -do - ifconfig -a | grep eth$i - if [ $? -eq 0 ]; - then - ifconfig eth$i up - dhclient eth$i -v - fi -done - -#mount config drive -mkdir -p /mnt/config -mount /dev/disk/by-label/config-2 /mnt/config - -# Facter fqdn will come from DNS unless we do this -echo "127.0.1.1 $(hostname).%{domain} $(hostname)" >> /etc/hosts diff --git a/stack-builder/fragments/stack-wait-build b/stack-builder/fragments/stack-wait-build deleted file mode 100644 index 251a8f8..0000000 --- a/stack-builder/fragments/stack-wait-build +++ /dev/null @@ -1,4 +0,0 @@ -until [ $(curl http://%{cobbler_node_ip}/status | grep up) ]; do - echo "waited for build" >> /root/waiting - sleep 1 -done diff --git a/stack-builder/fragments/test-nova b/stack-builder/fragments/test-nova deleted file mode 100644 index 978949c..0000000 --- a/stack-builder/fragments/test-nova +++ /dev/null @@ -1,9 +0,0 @@ -# This gives the controller access to the floating network -# without entering the namespace -ifconfig br-ex 172.16.2.1 netmask 255.255.255.0 up - -# allow compute nodes to come up -sleep 20 - -# This is a basic test of nova, keystone and quantum functionality -/tmp/test_nova.sh diff --git a/stack-builder/hiera_config.py b/stack-builder/hiera_config.py deleted file mode 100644 index 36644e7..0000000 --- a/stack-builder/hiera_config.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" - stack-builder.hiera_config - ~~~~~~~~~~~~~~~~~~~~~~~~~~ - - This module will read metadata set during instance - launch and override any yaml under the /etc/puppet/data - directory (except data_mappings) that has a key matching - the metadata -""" - -import yaml -import os - -hiera_dir = '/etc/puppet/data' -metadata_path = '/root/config.yaml' - -#debug -#metadata_path = './sample.json' -#hiera_dir = './openstack-installer/data/' - -# Child processes cannot set environment variables, so -# create a bash file to set some exports for facter -def facter_config(): - with open(metadata_path, 'r') as metadata: - meta = yaml.load(metadata.read()) - print meta - with open('/root/fact_exports', 'w') as facts: - for key,value in meta.items(): - facts.write('FACTER_' + str(key) + '=' + str(value) + '\n') - -#TODO -def hostname_config(): - with open(metadata_path, 'r') as metadata: - meta = yaml.load(metadata.read()) - with open('/root/openstack-installer/manifests/setup.pp', 'a') as facts: - for key,value in meta.items(): - pass - -facter_config() diff --git a/stack-builder/metadata.py b/stack-builder/metadata.py deleted file mode 100644 index b114f97..0000000 --- a/stack-builder/metadata.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -""" - stack-builder.metadata - ~~~~~~~~~~~~~~~~~~~~~~ - - This module will load in relevant environment variables - and config from the scenario yaml in order to create a - dictionary of metadata that will be used to build shell - scripts, populate hiera data for puppet, and drive the - creation of appropriate openstack resources for the - specified scenario. Environment variables will - override yaml data. -""" -import os -import yaml - -def import_environ_keys(metadata, prefix): - """ - Import any environment variables with the correct - prefix into the metadata dictionary - """ - for key,value in os.environ.items(): - if key[:9] == prefix: - metadata[key[9:]] = value - return metadata - -def import_yaml(path, files): - """ - """ - metadata = {} - - for filename in files: - if os.path.exists(path+filename+'.yaml'): - with open(path+filename+'.yaml', 'r') as f: - y = yaml.load(f.read()) - if y: - for key, value in y.items(): - metadata[key] = value - return metadata - -def build_metadata(path, scenario, config): - """ - Create a metadata dictionary from yaml - and environment variables - """ - if config == "config": - prefix = 'osi_conf_' - files = ['config'] - return import_environ_keys(import_yaml(path+'/', files), prefix) - if config == 'user': - prefix = 'osi_user_' - files = ['user'] - return import_environ_keys(import_yaml(path+'/hiera_data/',files), prefix) - if config == "global": - prefix = 'osi_glob_' - files = ['user'] - return import_environ_keys(import_yaml(path+'/global_hiera_params/', files), prefix) - else: - print "Invalid config type: choose from 'user', 'conf' and 'glob'" - -def show(n, q, k, args): - hostname = args.node - yaml_dir = args.yaml_dir - scenario = args.scenario - config = args.config - - print yaml.dump(build_metadata(yaml_dir, scenario, config), default_flow_style=False) diff --git a/stack-builder/rebuild.py b/stack-builder/rebuild.py deleted file mode 100644 index 4fa0790..0000000 --- a/stack-builder/rebuild.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python -""" - stack-builder.rebuild - ~~~~~~~~~~~~~~~~~~~ - This module will rebuild any unbuilt nodes - in a currently running test - -""" -import debug -import subprocess -import os -import quantumclient -import fragment -import yaml -import json -import time -import build - -from metadata import build_metadata -from debug import dprint - -# Assumes the networks hasn't been killed -def rebuild(n,q,k,args): - image = args.image - ci_subnet_index = 123 # TODO fix inital setup stuff - scenario = args.scenario - data_path = args.data_path - fragment_path = args.fragment_path - test_id = args.test_id - - if args.debug: - debug.debug = True - - with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: - scenario_yaml = yaml.load(scenario_yaml_file.read()) - - current_instances = build.get(n,q,k,args)[unicode(test_id)] - print current_instances - # determine which instances are missing - current_instance_names = [instance.name for instance in current_instances] - missing_instance_names = [] - for instance in scenario_yaml['nodes'].keys(): - if instance not in current_instance_names: - missing_instance_names.append(instance) - - dprint("present instances: " + str(current_instance_names)) - dprint("missing instances: " + str(missing_instance_names)) - networks = {} - subnets = {} - ports = {} - - networks['ci'] = build.get_ci_network(q,k) - - all_nets = q.list_networks() - - # build networks list, assume they're all there - for node, props in scenario_yaml['nodes'].items(): - for network in props['networks']: - if network != 'ci': - networks[network] = get_network(all_nets, network, test_id) - - dprint('networks ' + str(networks)) - # New ports for the missing instances - for node, props in scenario_yaml['nodes'].items(): - if node in missing_instance_names: - for network in props['networks']: - if node not in ports: - ports[node] = {} - dprint('creating port for node' + str(node) + ' on network ' + network) - ports[node][network] = build.allocate_ports(q, networks[network]['id'], test_id) - else: - dprint('creating port for node' + str(node) + ' on network ' + network) - ports[node][network] = build.allocate_ports(q, networks[network]['id'], test_id) - - # Recreate port dictionary for existing instances - for node in current_instances: - nodename = str(node.name) - for network, ips in node.networks.items(): - if nodename not in ports: - ports[nodename] = {} - ports[nodename][network] = [{'fixed_ips' : [{'ip_address' : ips[0]}]}] - else: - ports[nodename][network] = [{'fixed_ips' : [{'ip_address' : ips[0]}]}] - - dprint("Ports" + str(ports)) - # Re-create metadata and deploy - initial_config_meta = build_metadata(data_path, scenario, 'config') - hiera_config_meta = build_metadata(data_path, scenario, 'user') - - meta_update = build.metadata_update(scenario_yaml, ports) - dprint('runtime metadata: ' + str(meta_update)) - hiera_config_meta.update(meta_update) - initial_config_meta.update(meta_update) - - # fragment composition - deploy_files = {} - for node, props in scenario_yaml['nodes'].items(): - deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) - dprint(node + 'deploy:\n' + deploy_files[node]) - - user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) - initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) - - # Create missing instances - port_list = {} - for node, props in scenario_yaml['nodes'].items(): - if node in missing_instance_names: - nics = [] - for network in props['networks']: - nics.append(ports[node][network][0]['id']) - port_list[node] = build.build_nic_port_list(nics) - - for node, props in scenario_yaml['nodes'].items(): - if node in missing_instance_names: - dprint("booting " + str(node)) - build.boot_puppetised_instance(n, - node, - image, - port_list[node], - deploy=build.cloud_init, - files={ - u'/root/deploy' : deploy_files[node], - u'/root/user.yaml' : user_config_yaml, - u'/root/config.yaml' : initial_config_yaml}, - meta={'ci_test_id' : test_id} - ) - -def get_network(all_nets, network, test_id): - for n in all_nets['networks']: - if n['name'] == unicode('ci-'+network+'-'+test_id): - return n