#!/usr/bin/env python import os, sys, re, json, requests, base64 from subprocess import call from fabric import api from fabric.api import cd, run, put, sudo from os import environ as env from datetime import datetime from time import sleep # Remove SSH private key as it needs more processing CONFIG = json.loads(re.sub(r'("DOCKER_CI_KEY".+?"(.+?)",)','', env['CONFIG_JSON'], flags=re.DOTALL)) # Populate environment variables for key in CONFIG: env[key] = CONFIG[key] # Load SSH private key env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1', env['CONFIG_JSON'],flags=re.DOTALL) DROPLET_NAME = env.get('DROPLET_NAME','docker-ci') TIMEOUT = 120 # Seconds before timeout droplet creation IMAGE_ID = 1004145 # Docker on Ubuntu 13.04 REGION_ID = 4 # New York 2 SIZE_ID = 62 # memory 2GB DO_IMAGE_USER = 'root' # Image user on Digital Ocean API_URL = 'https://api.digitalocean.com/' DOCKER_PATH = '/go/src/github.com/dotcloud/docker' DOCKER_CI_PATH = '/docker-ci' CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH) class DigitalOcean(): def __init__(self, key, client): '''Set default API parameters''' self.key = key self.client = client self.api_url = API_URL def api(self, cmd_path, api_arg={}): '''Make api call''' api_arg.update({'api_key':self.key, 'client_id':self.client}) resp = requests.get(self.api_url + cmd_path, params=api_arg).text resp = json.loads(resp) if resp['status'] != 'OK': raise Exception(resp['error_message']) return resp def droplet_data(self, name): '''Get droplet data''' data = self.api('droplets') data = [droplet for droplet in data['droplets'] if droplet['name'] == name] return data[0] if data else {} def json_fmt(data): '''Format json output''' return json.dumps(data, sort_keys = True, indent = 2) do = DigitalOcean(env['DO_API_KEY'], env['DO_CLIENT_ID']) # Get DROPLET_NAME data data = do.droplet_data(DROPLET_NAME) # Stop processing if DROPLET_NAME exists on Digital Ocean if data: print ('Droplet: {} already deployed. Not further processing.' .format(DROPLET_NAME)) exit(1) # Create droplet do.api('droplets/new', {'name':DROPLET_NAME, 'region_id':REGION_ID, 'image_id':IMAGE_ID, 'size_id':SIZE_ID, 'ssh_key_ids':[env['DOCKER_KEY_ID']]}) # Wait for droplet to be created. start_time = datetime.now() while (data.get('status','') != 'active' and ( datetime.now()-start_time).seconds < TIMEOUT): data = do.droplet_data(DROPLET_NAME) print data['status'] sleep(3) # Wait for the machine to boot sleep(15) # Get droplet IP ip = str(data['ip_address']) print 'droplet: {} ip: {}'.format(DROPLET_NAME, ip) # Create docker-ci ssh private key so docker-ci docker container can communicate # with its EC2 instance os.makedirs('/root/.ssh') open('/root/.ssh/id_rsa','w').write(env['DOCKER_CI_KEY']) os.chmod('/root/.ssh/id_rsa',0600) open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n') api.env.host_string = ip api.env.user = DO_IMAGE_USER api.env.key_filename = '/root/.ssh/id_rsa' # Correct timezone sudo('echo "America/Los_Angeles" >/etc/timezone') sudo('dpkg-reconfigure --frontend noninteractive tzdata') # Load public docker-ci key sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB'])) # Create docker nightly release credentials file credentials = { 'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'], 'AWS_SECRET_KEY': env['PKG_SECRET_KEY'], 'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE']} open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write( base64.b64encode(json.dumps(credentials))) # Transfer docker sudo('mkdir -p ' + DOCKER_CI_PATH) sudo('chown {}.{} {}'.format(DO_IMAGE_USER, DO_IMAGE_USER, DOCKER_CI_PATH)) call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, DO_IMAGE_USER, ip, os.path.dirname(DOCKER_CI_PATH)), shell=True) # Install Docker and Buildbot dependencies sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker') sudo('apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9') sudo('echo deb https://get.docker.io/ubuntu docker main >' ' /etc/apt/sources.list.d/docker.list') sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n' 'deb http://us.archive.ubuntu.com/ubuntu/ raring-security main universe\n"' ' > /etc/apt/sources.list; apt-get update') sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev' ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)' ' aufs-tools make libfontconfig libevent-dev libsqlite3-dev libssl-dev') sudo('wget -O - https://go.googlecode.com/files/go1.2.linux-amd64.tar.gz | ' 'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go') sudo('GOPATH=/go go get -d github.com/dotcloud/docker') sudo('pip install -r {}/requirements.txt'.format(CFG_PATH)) # Install docker and testing dependencies sudo('apt-get install -y -q lxc-docker') sudo('curl -s https://phantomjs.googlecode.com/files/' 'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin' ' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs') # Build docker-ci containers sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH)) sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH)) sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format( DOCKER_CI_PATH)) sudo('cd {}/registry-coverage; docker build -t registry_coverage .'.format( DOCKER_CI_PATH)) # Download docker-ci testing container sudo('docker pull mzdaniel/test_docker') # Setup buildbot sudo('mkdir /data') sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}' ' {11} {12}'.format(CFG_PATH, DOCKER_PATH, env['BUILDBOT_PWD'], env['IRC_PWD'], env['IRC_CHANNEL'], env['SMTP_USER'], env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'], env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'], env['REGISTRY_SECRET_KEY'])) # Preventively reboot docker-ci daily sudo('ln -s /sbin/reboot /etc/cron.daily')