From fb60f7f6c1ea4152f05e11b4fb45c8391d5df1cc Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Wed, 5 Oct 2016 11:42:06 +0200 Subject: [PATCH 1/2] [INFRA-317] rewrite to use Ansible --- Dockerfile | 69 ++----------------- README.md | 4 +- circle.yml | 8 ++- create_python_image.sh | 13 ++++ image/environment | 1 + image/id_rsa | 51 ++++++++++++++ image/id_rsa.pub | 1 + provision-python-codebox.yml | 5 ++ .../files/external_requirements.txt | 1 - python/files/pkglist | 15 ++++ .../files/requirements.txt | 0 .../files/requirements_base.txt | 0 .../files/requirements_python3.txt | 0 .../files/requirements_v42.txt | 0 python/tasks/common.yml | 38 ++++++++++ python/tasks/envs.yml | 43 ++++++++++++ python/tasks/main.yml | 24 +++++++ 17 files changed, 204 insertions(+), 69 deletions(-) create mode 100755 create_python_image.sh create mode 100644 image/environment create mode 100644 image/id_rsa create mode 100644 image/id_rsa.pub create mode 100644 provision-python-codebox.yml rename external_requirements.txt => python/files/external_requirements.txt (98%) create mode 100644 python/files/pkglist rename requirements.txt => python/files/requirements.txt (100%) rename requirements_base.txt => python/files/requirements_base.txt (100%) rename requirements_python3.txt => python/files/requirements_python3.txt (100%) rename requirements_v42.txt => python/files/requirements_v42.txt (100%) create mode 100644 python/tasks/common.yml create mode 100644 python/tasks/envs.yml create mode 100644 python/tasks/main.yml diff --git a/Dockerfile b/Dockerfile index 8540e86..846fd37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,66 +1,9 @@ FROM ubuntu:trusty MAINTAINER "Syncano DevOps Team" -ENV LAST_REFRESHED 2016-03-25 -ENV SYNCANO_APIROOT https://api.syncano.io/ - -RUN groupadd -r syncano && \ - useradd -u 1000 -r -g syncano syncano -d /tmp -s /bin/bash && \ - mkdir /home/syncano && \ - chown -R syncano /home/syncano - -# enable everyone to use /tmp -RUN chmod 1777 /tmp -# -- CUT BEGIN -- - -COPY *requirements*.txt /tmp/ -COPY *.tar.gz /tmp/ - -RUN apt-get update && apt-get install -qqy \ - git \ - libffi-dev \ - libssl-dev \ - libjpeg-dev \ - python-dev \ - python3-dev \ - python-tk \ - python3-tk \ - python-numpy \ - python3-numpy \ - python-scipy \ - python3-scipy \ - wget && \ - wget https://bootstrap.pypa.io/get-pip.py && \ - python get-pip.py && \ - pip install -r /tmp/requirements_base.txt - -WORKDIR /home/syncano/ -RUN virtualenv --system-site-packages v4.2 && \ - tar xzvf /tmp/42.tar.gz --skip-old-files && \ - . v4.2/bin/activate && \ - pip install -r /tmp/requirements_v42.txt && \ - pip install -r /tmp/external_requirements.txt && \ - deactivate - -RUN virtualenv --system-site-packages v5.0 && \ - tar xzvf /tmp/50.tar.gz --skip-old-files && \ - . v5.0/bin/activate && \ - pip install -r /tmp/requirements.txt && \ - pip install -r /tmp/external_requirements.txt && \ - deactivate - -RUN virtualenv --system-site-packages -p python3 p3v5.0 && \ - . p3v5.0/bin/activate && \ - pip install -r /tmp/requirements_python3.txt && \ - pip install -r /tmp/external_requirements.txt - -RUN ln -sf /home/syncano/v5.0/bin/python /usr/bin/python && \ - ln -sf /home/syncano/v5.0/bin/python /usr/bin/python27-lib5.0 && \ - ln -sf /home/syncano/v4.2/bin/python /usr/bin/python27-lib4.2 && \ - ln -sf /home/syncano/p3v5.0/bin/python /usr/bin/python3 && \ - ln -sf /home/syncano/p3v5.0/bin/python /usr/bin/python3-lib5.0 - -# -- CUT END -- -USER syncano -WORKDIR /tmp -CMD "python" +RUN apt-get update && apt-get install -y openssh-server +RUN mkdir /var/run/sshd /root/.ssh +COPY image/id_rsa.pub /root/.ssh/authorized_keys +RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.0.3/dumb-init_1.0.3_amd64.deb +RUN dpkg -i dumb-init_*.deb +EXPOSE 22 diff --git a/README.md b/README.md index 9d6f9c1..b9b6be7 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ This repository contains Dockerfile for Python 2.7 image with built-in support for Syncano and selected Python libraries. -You can build the image yourself: +You can build the image yourself (Ansible 2.1.0 is required): ``` -$ docker build -t quay.io/syncano/python-codebox . +$ ./create_python_image.sh ``` or pull it from Docker registry: diff --git a/circle.yml b/circle.yml index 2ea99f5..6f7c766 100644 --- a/circle.yml +++ b/circle.yml @@ -4,9 +4,11 @@ machine: dependencies: override: - - wget https://s3.amazonaws.com/codebox-bucket/python/42.tar.gz - - wget https://s3.amazonaws.com/codebox-bucket/python/50.tar.gz - - docker build -t quay.io/syncano/python-codebox . + - sudo pip install ansible==2.1.0 + - chmod 600 image/id_rsa + - ./create_python_image.sh $BUILD_LEVEL + - docker images + - docker history quay.io/syncano/python-codebox test: override: diff --git a/create_python_image.sh b/create_python_image.sh new file mode 100755 index 0000000..0a12684 --- /dev/null +++ b/create_python_image.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ $1 != "--partial" ] +then + docker build -t quay.io/syncano/python-codebox . +fi + +docker run -d -p 2200:22 --env-file image/environment --name ansible quay.io/syncano/python-codebox dumb-init /usr/sbin/sshd -D +ansible-playbook -i "`docker inspect --format '{{ .NetworkSettings.IPAddress }}' ansible`," -u root --private-key image/id_rsa provision-python-codebox.yml +docker commit ansible quay.io/syncano/python-codebox + +docker run -d --env-file image/environment -u syncano --name python quay.io/syncano/python-codebox dumb-init /usr/sbin/sshd -D +docker commit python quay.io/syncano/python-codebox diff --git a/image/environment b/image/environment new file mode 100644 index 0000000..7228805 --- /dev/null +++ b/image/environment @@ -0,0 +1 @@ +SYNCANO_APIROOT=https://api.syncano.io/ diff --git a/image/id_rsa b/image/id_rsa new file mode 100644 index 0000000..1f1b49a --- /dev/null +++ b/image/id_rsa @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKgIBAAKCAgEAqdVP3R97ex0midPnCFkya/AWZG+tNDjHlHarTwEWjenxYfJu +eJTpUCxCFVomhVQVwdHQ9egdyy6KlsxC7fhPKNkLkkv+rCNk8Fn4m3NjmIv5eGN/ +YxpojxjNivfxK0PaTB9FrnZmJQNW/vLPuq92gl2OeNHfwIx6B4/yCxQtN1hTfA5f +6DKWorD874Z32pwJiJEnNygQXoHfXZdv4+pmbOLAKYjKMwNm+epy+CD0rwrChm/K +b9hcV9m9Z5s0qW5U4cqPBqt3QZghKwsj1g0rYQc0pHFg50NvWOQePJ3GCxstVfy5 +uAk7HIQj15xncb68xWmFaZLLYPxLeg4LMdEsTwejxpob1p1vtCGLILVFTilxQCOx +fJCRXaNDPQMwxHRyTSvJ5X6JvXlvs7bWsK3JUGnmzSa4s9B/N44/OrNjLW37AVx7 +j7Bo1GbBoqNfAYZ3GYvKfBTIcZ5fSUwH5A7mDSPDgBPLyUOJUg4ymZpyKilZASIV +Tp9weGU0BMoHnIFBag/Jd66Y98jCWjpHvCGKQUD9eIFUc7TRwtiI8VVD3+Fi6Yc9 +S8lCB1O5EiLD4os/Dmq1+F/YiHzTWnPEpVxuZw+vyzzQvzgroetHK3kWGwn9iDB2 +ABKin1GBjrQpWc0sH5au6NX0dpDL1RfaXHRk/m96kr6ZLTBeA9Z6gwsrTIMCAwEA +AQKCAgB/ZsipfXN5MK33WOIITjlX5Vw27dvFwA5f3gG2m39a6DMuodmjVToM7ZaJ +Q/5/qEtGAWq6/exTvk8yvno64Dopz2Ax2BZs9YREFYo8uvvstJxpZmJqaqYjKp0O +cd8V8KmyzwbL0H/hpja9vFJ+nh/NvfchxXcPj5H/WElfbDxbN0QxXOsvWbvMwe6M +lJgWg1VXgGQlqUVWUCDKqc7C79T0O8mhnr7oM9YtBmZGoxpKsUyp4vS+oFTthibi +sPh5XY4Jb5WJkFk++KtxCcNRUMHI4Mnpzi++bMT7ruzL1iN5s2J4DcUYPLx+MNZe +aTLkjgw4H2d4gilVUQ4huqVFMsAEM6WvB1a8mr0fXu0lHXmjIEVFS9wDX63FD8kR +TwrQMNn4hzZWGCj2i3vpkmzGPHkPK+rYQ61lkHyvN5IMTEsBIdvK6c09vMSIE8oc +Xm1X3aRShjgkVmRgrTKiLInHW1cKzVDANOW8jHx0g7IqQNe0ap899IyJsKcA4cXJ +fkRhyu9D0pxvkK6Jp65QR84yQ+qYDQQ+cVopg7PFtkj2Ek9s5Glq4FXEZXN1RU2Q +SlFm6eP3g+whMg+8hTBmnaBe0KmH7biJAGTTnL2xzCayXM/PtqrWOlCqtDoB1JVk +13r1P0+B4lPBitw2WXgEGk/neNt3Og2cLajsnT5oNGgdIjrFYQKCAQEA3horKk5U +fRbFRSBf5tWTRsoJrJ5F0ygp921ehhFHAH636eoDz8FuS/ZrAiLoIctmhBSY3H0b +cxpY2WiA3vao/GrBatQ+fi5mlwckIzCUoStHotVOisUl9AXhsUBjBYLsDYB8k1ni +89ZZC5x3XMdzoAy995U8b1Fcxyl5txuI8nlAOyPZf896tbKQ4zs/pLktaEetpo6W +QF7pgWGxfOsFBuej3q2AXm8h5Aw4xFv3LHP5KePM6m/xwZjsyzvNLWo3f0hRrLS/ +RYb4fzwfZ9WMe/Mh0Uv3a0r3k+Orpv159+8UDEqijpFo1hvmKU3q6fGVkxt5/J5y +GG1vZKYF3zDs0wKCAQEAw8Dssx327QWSKFRiCF9oWivWHGv+rQidjQd2MEkQF/9N +ab8cfBXNzEo9/jMcXtwa+bULDTuKtlTGsIqeXYrszRnFs91RYjSnrqbOFZJAHoxJ +/rBgvQjAmDddDyeRAaW1FU5U0uvqudDjS6ELBFIN79rXj5xJlgcC88C9nUmmHfaM +1HlIfN+LXJ5H5PkNgbD8ZT1ZJH4rEtlkPx5lNdgrShchrZ6ll13fLa+EbMOB0HNl +vFuQw1mMJpm3IEoAh1P/iXyWDpJRKTvKtld3AR0/4JOapqIUas+CIaaF/VZhwsUY +0dn+YASi2+bgWvI5VKfoaF9LGrws+oY8mxxdqqiTkQKCAQEAnalB3wQ/oZRacNms +ejY+F9Av0eOAeYZXVBbFNLJrDoTllRzTm9UZsL6584hH9EVo/nWIlWwR4kFftOUk +JiI+jwBRUL5dpkDV/nvJQVmpCEeq0IJPAN82M+VA8wxbvnvTiToeNJNoMeOEzmNB +sBC9mW2yJGBByFufWpmslzjqnAVKfTDTopDr8LNTepqCcTIqc43+TO+G5Sb65ycs +URqcNOyWPOo459BH5JRwb6NA8MNaqkvVGM5idSVTYmmPGp/9yVLS96BO7tIC8H8D +ZDgJQ6ux49rjRbyvxsGlBiOFQ2TpfFg20LGUE/k2BzgSfANMRGMK0zK+l2yH2JCq +NmeKlQKCAQEAugsLaxiZa58Vz7qTCT4T4U80SQLiovVDN5vV2gI2h3gezNfTbKMU +Wopc7NEKr6UlBhnojQT/ylY8F+FGrcSxViSeX1mCVYGRtE3YI2xCAEZ4tNRQvJgq +4wALM7H5AMXKmps9xtBs33kF7QT7gXmywTkOEfEJhsrXJW8TRv+OH4AJTI6QNA4J +V1cfISEKky3wUw/BPwuiSGHQAqUSr9/pFJtzaJ0U6A1RZQwhIXWWVNS35hJoPSFI +MA97mfZ7FiLhBHwh6WtqM3QeLrpl4es4oGnoW2JDYVgiZ3Tqd2G3u/Kyqg8y0XLW +jYSZyhXYIvsJyNh4espJPTiDNS/6XY2l4QKCAQEAzzRobY+LOLJUt1HAQdbJEeXS +4EH3vn7vPoxME56bSPvnz/w/u8RSh1mvnMx41ANYccUsNBbBdyZClG3UAToAxuxu +Z9j2ptNPFIlhZxGgf7o3p7hCU0DosMZVR1ADNJb+tGgXjHwctYIbX9Prbf30y06l +G3XlStZE0rVeC9RnM/OD4ymaXfp/wRsEGMX1vGhdsFebRFCauuimvRDOw8bRhUbI +oWXCUVSP5PqY/io0QAFZgCgi9OgtHO8qKzzK98bWPprHBXUGDWnuzXB/irbiQhBJ +tE6FDAh56fnBOxiYjdvh5iKv6rbuT3esoWe4S9H7vNjI7kHer95/tAsqVseTag== +-----END RSA PRIVATE KEY----- diff --git a/image/id_rsa.pub b/image/id_rsa.pub new file mode 100644 index 0000000..11354bd --- /dev/null +++ b/image/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCp1U/dH3t7HSaJ0+cIWTJr8BZkb600OMeUdqtPARaN6fFh8m54lOlQLEIVWiaFVBXB0dD16B3LLoqWzELt+E8o2QuSS/6sI2TwWfibc2OYi/l4Y39jGmiPGM2K9/ErQ9pMH0WudmYlA1b+8s+6r3aCXY540d/AjHoHj/ILFC03WFN8Dl/oMpaisPzvhnfanAmIkSc3KBBegd9dl2/j6mZs4sApiMozA2b56nL4IPSvCsKGb8pv2FxX2b1nmzSpblThyo8Gq3dBmCErCyPWDSthBzSkcWDnQ29Y5B48ncYLGy1V/Lm4CTschCPXnGdxvrzFaYVpkstg/Et6Dgsx0SxPB6PGmhvWnW+0IYsgtUVOKXFAI7F8kJFdo0M9AzDEdHJNK8nlfom9eW+zttawrclQaebNJriz0H83jj86s2MtbfsBXHuPsGjUZsGio18BhncZi8p8FMhxnl9JTAfkDuYNI8OAE8vJQ4lSDjKZmnIqKVkBIhVOn3B4ZTQEygecgUFqD8l3rpj3yMJaOke8IYpBQP14gVRztNHC2IjxVUPf4WLphz1LyUIHU7kSIsPiiz8OarX4X9iIfNNac8SlXG5nD6/LPNC/OCuh60creRYbCf2IMHYAEqKfUYGOtClZzSwflq7o1fR2kMvVF9pcdGT+b3qSvpktMF4D1nqDCytMgw== syncano diff --git a/provision-python-codebox.yml b/provision-python-codebox.yml new file mode 100644 index 0000000..62fdb24 --- /dev/null +++ b/provision-python-codebox.yml @@ -0,0 +1,5 @@ +--- +- hosts: all + connection: ssh + roles: + - python diff --git a/external_requirements.txt b/python/files/external_requirements.txt similarity index 98% rename from external_requirements.txt rename to python/files/external_requirements.txt index b3826da..8788358 100644 --- a/external_requirements.txt +++ b/python/files/external_requirements.txt @@ -1,3 +1,2 @@ -i https://pypi.fury.io/Nx-poGuiakLbsCzaLwNm/aexol aexol==0.1.5 - diff --git a/python/files/pkglist b/python/files/pkglist new file mode 100644 index 0000000..7c1c2ee --- /dev/null +++ b/python/files/pkglist @@ -0,0 +1,15 @@ +git +libffi-dev +libssl-dev +libjpeg-dev +python-dev +python3-dev +python-tk +python3-tk +python-numpy +python3-numpy +python-pip +python3-pip +python-scipy +python3-scipy +wget diff --git a/requirements.txt b/python/files/requirements.txt similarity index 100% rename from requirements.txt rename to python/files/requirements.txt diff --git a/requirements_base.txt b/python/files/requirements_base.txt similarity index 100% rename from requirements_base.txt rename to python/files/requirements_base.txt diff --git a/requirements_python3.txt b/python/files/requirements_python3.txt similarity index 100% rename from requirements_python3.txt rename to python/files/requirements_python3.txt diff --git a/requirements_v42.txt b/python/files/requirements_v42.txt similarity index 100% rename from requirements_v42.txt rename to python/files/requirements_v42.txt diff --git a/python/tasks/common.yml b/python/tasks/common.yml new file mode 100644 index 0000000..1f4fcb0 --- /dev/null +++ b/python/tasks/common.yml @@ -0,0 +1,38 @@ +--- +- name: Add syncano group + group: + name: syncano + gid: 1000 + state: present + +- name: Add syncano user + user: + name: syncano + group: syncano + uid: 1000 + shell: /bin/bash + home: /home/syncano + state: present + +- name: Copy apt pkgs + copy: + src: pkglist + dest: /tmp/ + +- name: List apt packages + shell: + cat /tmp/pkglist + register: pkgs + +- name: Install packages + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + "{{ pkgs.stdout_lines }}" + +- name: Allow public usage of /tmp + file: + name: /tmp + mode: "1777" \ No newline at end of file diff --git a/python/tasks/envs.yml b/python/tasks/envs.yml new file mode 100644 index 0000000..e35faad --- /dev/null +++ b/python/tasks/envs.yml @@ -0,0 +1,43 @@ +- name: Install Python packages in envs + pip: + virtualenv: "/home/syncano/{{ item.env }}" + virtualenv_python: "{{ item.python }}" + virtualenv_site_packages: yes + requirements: "/tmp/{{ item.requirements }}" + with_items: + - env: v4.2 + python: python2.7 + requirements: requirements_v42.txt + - env: v4.2 + python: python2.7 + requirements: external_requirements.txt + - env: v5.0 + python: python2.7 + requirements: requirements.txt + - env: v5.0 + python: python2.7 + requirements: external_requirements.txt + - env: p3v5.0 + python: python3 + requirements: requirements_python3.txt + - env: p3v5.0 + python: python3 + requirements: external_requirements.txt + +- name: Set soft links to runtimes + file: + src: "/home/syncano/{{ item.env }}/bin/python" + dest: "/usr/bin/{{ item.python }}" + state: link + force: yes + with_items: + - env: v5.0 + python: python + - env: v5.0 + python: python27-lib5.0 + - env: v4.2 + python: python27-lib4.2 + - env: p3v5.0 + python: python3 + - env: p3v5.0 + python: python3-lib5.0 diff --git a/python/tasks/main.yml b/python/tasks/main.yml new file mode 100644 index 0000000..d7cb095 --- /dev/null +++ b/python/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- include: common.yml + +- name: Install requirements + pip: + name: pip + extra_args: "--upgrade" + +- name: Copy requirements files + copy: + src: "{{ item }}" + dest: /tmp/ + with_items: + - requirements.txt + - requirements_base.txt + - requirements_v42.txt + - requirements_python3.txt + - external_requirements.txt + +- name: Install requirements + pip: + requirements: /tmp/requirements_base.txt + +- include: envs.yml From 7b5167449f73d1c7a3ffd59db9f36ed168156def Mon Sep 17 00:00:00 2001 From: Zhe Brah Date: Fri, 7 Oct 2016 11:44:25 +0200 Subject: [PATCH 2/2] [INFRA-317] correct default param [ci skip] --- create_python_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_python_image.sh b/create_python_image.sh index 0a12684..e66953b 100755 --- a/create_python_image.sh +++ b/create_python_image.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ $1 != "--partial" ] +if [ ${1:-full} != "--partial" ] then docker build -t quay.io/syncano/python-codebox . fi