docker-ci 0.40. Migrate docker-ci to Digital Ocean.

hack/infrastructure/docker-ci/Dockerfile

@@ -1,14 +1,16 @@
 # VERSION:        0.22
 # DOCKER-VERSION  0.6.3
 # AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
-# DESCRIPTION:    Deploy docker-ci on Amazon EC2
+# DESCRIPTION:    Deploy docker-ci on Digital Ocean
 # COMMENTS:
 #     CONFIG_JSON is an environment variable json string loaded as:
 #
 # export CONFIG_JSON='
-#     { "AWS_TAG":             "EC2_instance_name",
+#     { "DROPLET_NAME":        "docker-ci",
-#       "AWS_ACCESS_KEY":      "EC2_access_key",
+#       "DO_CLIENT_ID":        "Digital_Ocean_client_id",
-#       "AWS_SECRET_KEY":      "EC2_secret_key",
+#       "DO_API_KEY":          "Digital_Ocean_api_key",
+#       "DOCKER_KEY_ID":       "Digital_Ocean_ssh_key_id",
+#       "DOCKER_CI_KEY_PATH":  "docker-ci_private_key_path",
 #       "DOCKER_CI_PUB":       "$(cat docker-ci_ssh_public_key.pub)",
 #       "DOCKER_CI_KEY":       "$(cat docker-ci_ssh_private_key.key)",
 #       "BUILDBOT_PWD":        "Buildbot_server_password",
@@ -37,7 +39,7 @@ run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' \
     > /etc/apt/sources.list
 run apt-get update; apt-get install -y git python2.7 python-dev libevent-dev \
     python-pip ssh rsync less vim
-run pip install boto fabric
+run pip install requests fabric
 
 # Add deployment code and set default container command
 add . /docker-ci

hack/infrastructure/docker-ci/deployment.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 
-import os, sys, re, json, base64
+import os, sys, re, json, requests, base64
-from boto.ec2.connection import EC2Connection
 from subprocess import call
 from fabric import api
 from fabric.api import cd, run, put, sudo
 from os import environ as env
+from datetime import datetime
 from time import sleep
 
 # Remove SSH private key as it needs more processing
@@ -20,42 +20,41 @@ for key in CONFIG:
 env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1',
     env['CONFIG_JSON'],flags=re.DOTALL)
 
-
+DROPLET_NAME = env.get('DROPLET_NAME','docker-ci')
-AWS_TAG = env.get('AWS_TAG','docker-ci')
+TIMEOUT = 120            # Seconds before timeout droplet creation
-AWS_KEY_NAME = 'dotcloud-dev'       # Same as CONFIG_JSON['DOCKER_CI_PUB']
+IMAGE_ID = 1004145       # Docker on Ubuntu 13.04
-AWS_AMI = 'ami-d582d6bc'            # Ubuntu 13.04
+REGION_ID = 4            # New York 2
-AWS_REGION = 'us-east-1'
+SIZE_ID = 62             # memory 2GB
-AWS_TYPE = 'm1.small'
+DO_IMAGE_USER = 'root'   # Image user on Digital Ocean
-AWS_SEC_GROUPS = 'gateway'
+API_URL = 'https://api.digitalocean.com/'
-AWS_IMAGE_USER = 'ubuntu'
 DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
 DOCKER_CI_PATH = '/docker-ci'
 CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH)
 
 
-class AWS_EC2:
+class digital_ocean():
-    '''Amazon EC2'''
+
-    def __init__(self, access_key, secret_key):
+    def __init__(self, key, client):
         '''Set default API parameters'''
-        self.handler = EC2Connection(access_key, secret_key)
+        self.key = key
-    def create_instance(self, tag, instance_type):
+        self.client = client
-        reservation = self.handler.run_instances(**instance_type)
+        self.api_url = API_URL
-        instance = reservation.instances[0]
+
-        sleep(10)
+    def api(self, cmd_path, api_arg={}):
-        while instance.state != 'running':
+        '''Make api call'''
-            sleep(5)
+        api_arg.update({'api_key':self.key, 'client_id':self.client})
-            instance.update()
+        resp = requests.get(self.api_url + cmd_path, params=api_arg).text
-            print "Instance state: %s" % (instance.state)
+        resp = json.loads(resp)
-        instance.add_tag("Name",tag)
+        if resp['status'] != 'OK':
-        print "instance %s done!" % (instance.id)
+            raise Exception(resp['error_message'])
-        return instance.ip_address
+        return resp
-    def get_instances(self):
+
-        return self.handler.get_all_instances()
+    def droplet_data(self, name):
-    def get_tags(self):
+        '''Get droplet data'''
-        return dict([(i.instances[0].id, i.instances[0].tags['Name'])
+        data = self.api('droplets')
-            for i in self.handler.get_all_instances() if i.instances[0].tags])
+        data = [droplet for droplet in data['droplets']
-    def del_instance(self, instance_id):
+            if droplet['name'] == name]
-        self.handler.terminate_instances(instance_ids=[instance_id])
+        return data[0] if data else {}
 
 
 def json_fmt(data):
@@ -63,20 +62,36 @@ def json_fmt(data):
     return json.dumps(data, sort_keys = True, indent = 2)
 
 
-# Create EC2 API handler
+do = digital_ocean(env['DO_API_KEY'], env['DO_CLIENT_ID'])
-ec2 = AWS_EC2(env['AWS_ACCESS_KEY'], env['AWS_SECRET_KEY'])
 
-# Stop processing if AWS_TAG exists on EC2
+# Get DROPLET_NAME data
-if AWS_TAG in ec2.get_tags().values():
+data = do.droplet_data(DROPLET_NAME)
-    print ('Instance: {} already deployed. Not further processing.'
+
-        .format(AWS_TAG))
+# Stop processing if DROPLET_NAME exists on Digital Ocean
+if data:
+    print ('Droplet: {} already deployed. Not further processing.'
+        .format(DROPLET_NAME))
     exit(1)
 
-ip = ec2.create_instance(AWS_TAG, {'image_id':AWS_AMI, 'instance_type':AWS_TYPE,
+# Create droplet
-    'security_groups':[AWS_SEC_GROUPS], 'key_name':AWS_KEY_NAME})
+do.api('droplets/new', {'name':DROPLET_NAME, 'region_id':REGION_ID,
+    'image_id':IMAGE_ID, 'size_id':SIZE_ID,
+    'ssh_key_ids':[env['DOCKER_KEY_ID']]})
 
-# Wait 30 seconds for the machine to boot
+# Wait for droplet to be created.
-sleep(30)
+start_time = datetime.now()
+while (data.get('status','') != 'active' and (
+ datetime.now()-start_time).seconds < TIMEOUT):
+    data = do.droplet_data(DROPLET_NAME)
+    print data['status']
+    sleep(3)
+
+# Wait for the machine to boot
+sleep(15)
+
+# Get droplet IP
+ip = str(data['ip_address'])
+print 'droplet: {}    ip: {}'.format(DROPLET_NAME, ip)
 
 # Create docker-ci ssh private key so docker-ci docker container can communicate
 # with its EC2 instance
@@ -86,7 +101,7 @@ os.chmod('/root/.ssh/id_rsa',0600)
 open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n')
 
 api.env.host_string = ip
-api.env.user = AWS_IMAGE_USER
+api.env.user = DO_IMAGE_USER
 api.env.key_filename = '/root/.ssh/id_rsa'
 
 # Correct timezone
@@ -106,13 +121,11 @@ open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write(
 
 # Transfer docker
 sudo('mkdir -p ' + DOCKER_CI_PATH)
-sudo('chown {}.{} {}'.format(AWS_IMAGE_USER, AWS_IMAGE_USER, DOCKER_CI_PATH))
+sudo('chown {}.{} {}'.format(DO_IMAGE_USER, DO_IMAGE_USER, DOCKER_CI_PATH))
-call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, AWS_IMAGE_USER, ip,
+call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, DO_IMAGE_USER, ip,
     os.path.dirname(DOCKER_CI_PATH)), shell=True)
 
 # Install Docker and Buildbot dependencies
-sudo('addgroup docker')
-sudo('usermod -a -G docker ubuntu')
 sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker')
 sudo('wget -q -O - https://get.docker.io/gpg | apt-key add -')
 sudo('echo deb https://get.docker.io/ubuntu docker main >'
@@ -122,7 +135,7 @@ sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n'
     ' > /etc/apt/sources.list; apt-get update')
 sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev'
     ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)'
-    ' aufs-tools make libfontconfig libevent-dev')
+    ' aufs-tools make libfontconfig libevent-dev libsqlite3-dev libssl-dev')
 sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | '
     'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go')
 sudo('GOPATH=/go go get -d github.com/dotcloud/docker')
@@ -137,9 +150,6 @@ sudo('curl -s https://phantomjs.googlecode.com/files/'
 # Preventively reboot docker-ci daily
 sudo('ln -s /sbin/reboot /etc/cron.daily')
 
-# Preventively reboot docker-ci daily
-sudo('ln -s /sbin/reboot /etc/cron.daily')
-
 # Build docker-ci containers
 sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH))
 sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH))