From 42697a4ecb2c034c9e88e245932b3914dfd1206c Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Fri, 13 Dec 2013 03:03:15 -0800 Subject: [PATCH 1/2] use S3-based node_modules caching --- .travis.yml | 10 +++- test-infra/node_modules_cache.py | 87 ++++++++++++++++++++++++++++++++ test-infra/requirements.txt | 1 + 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100755 test-infra/node_modules_cache.py create mode 100644 test-infra/requirements.txt diff --git a/.travis.yml b/.travis.yml index 7ccf23744c..2e71f553aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,18 @@ language: node_js node_js: - 0.10 -before_script: +before_install: + - sudo pip install --use-mirrors -r ./test-infra/requirements.txt +install: - gem install jekyll - npm install -g grunt-cli + - ./test-infra/node_modules_cache.py download || npm install +after_script: + - ./test-infra/node_modules_cache.py upload env: global: - SAUCE_USERNAME: bootstrap - secure: "pJkBwnuae9dKU5tEcCqccfS1QQw7/meEcfz63fM7ba7QJNjoA6BaXj08L5Z3Vb5vBmVPwBawxo5Hp0jC0r/Z/O0hGnAmz/Cz09L+cy7dSAZ9x4hvZePSja/UAusaB5ogMoO8l2b773MzgQeSmrLbExr9BWLeqEfjC2hFgdgHLaQ=" + - secure: "gqjqISbxBJK6byFbsmr1AyP1qoWH+rap06A2gI7v72+Tn2PU2nYkIMUkCvhZw6K889jv+LhQ/ybcBxDOXHpNCExCnSgB4dcnmYp+9oeNZb37jSP0rQ+Ib4OTLjzc3/FawE/fUq5kukZTC7porzc/k0qJNLAZRx3YLALmK1GIdUY=" + - secure: "Gghh/e3Gsbj1+4RR9Lh2aR/xJl35HWiHqlPIeSUqE9D7uDCVTAwNce/dGL3Ew7uJPfJ6Pgr70wD3zgu3stw0Zmzayax0hiDtGwcQCxVIER08wqGANK9C2Q7PYJkNTNtiTo6ehKWbdV4Z+/U+TEYyQfpQTDbAFYk/vVpsdjp0Lmc=" + - secure: "RTbRdx4G/2OTLfrZtP1VbRljxEmd6A1F3GqXboeQTldsnAlwpsES65es5CE3ub/rmixLApOY9ot7OPmNixFgC2Y8xOsV7lNCC62QVpmqQEDyGFFQKb3yO6/dmwQxdsCqGfzf9Np6Wh5V22QFvr50ZLKLd7Uhd9oXMDIk/z1MJ3o=" diff --git a/test-infra/node_modules_cache.py b/test-infra/node_modules_cache.py new file mode 100755 index 0000000000..9d9272fcc1 --- /dev/null +++ b/test-infra/node_modules_cache.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python2.7 +from __future__ import absolute_import, unicode_literals, print_function, division + +from sys import argv +from os import environ, stat, remove as _delete_file +from os.path import isfile +from hashlib import sha256 +from subprocess import check_call as run + +from boto.s3.connection import S3Connection +from boto.s3.key import Key +from boto.exception import S3ResponseError + + +NODE_MODULES_TARBALL = 'node_modules.tar.gz' +NEED_TO_UPLOAD_MARKER = '.need-to-upload' +BYTES_PER_MB = 1024 * 1024 +try: + BUCKET_NAME = environ['TWBS_S3_BUCKET'] +except KeyError: + raise SystemExit("TWBS_S3_BUCKET environment variable not set!") + + +def _sha256_of_file(filename): + hasher = sha256() + with open(filename, 'rb') as input_file: + hasher.update(input_file.read()) + return hasher.hexdigest() + + +def _delete_file_quietly(filename): + try: + _delete_file(filename) + except (OSError, IOError): + pass + + +def _tarball_size(): + kib = stat(NODE_MODULES_TARBALL).st_size // BYTES_PER_MB + return "{} MiB".format(kib) + + +if __name__ == '__main__': + # Uses environment variables: + # AWS_ACCESS_KEY_ID - AWS Access Key ID + # AWS_SECRET_ACCESS_KEY - AWS Secret Access Key + argv.pop(0) + if len(argv) != 1: + raise SystemExit("USAGE: node_modules_cache.py ") + mode = argv.pop() + + conn = S3Connection() + bucket = conn.lookup(BUCKET_NAME) + if bucket is None: + raise SystemExit("Could not access bucket!") + + package_json_hash = _sha256_of_file('package.json') + print('sha256(package.json) = ' + package_json_hash) + + key = Key(bucket, package_json_hash) + key.storage_class = 'REDUCED_REDUNDANCY' + + if mode == 'download': + _delete_file_quietly(NEED_TO_UPLOAD_MARKER) + try: + print("Downloading tarball from S3...") + key.get_contents_to_filename(NODE_MODULES_TARBALL) + except S3ResponseError as err: + open(NEED_TO_UPLOAD_MARKER, 'a').close() + print(err) + raise SystemExit("Cached node_modules download failed!") + print("Downloaded {}.".format(_tarball_size())) + print("Extracting tarball...") + run(['tar', 'xzf', NODE_MODULES_TARBALL]) + print("node_modules successfully installed from cache.") + elif mode == 'upload': + if isfile(NEED_TO_UPLOAD_MARKER): + print("Creating tarball...") + run(['tar', 'czf', NODE_MODULES_TARBALL, 'node_modules']) + print("Uploading tarball to S3... ({})".format(_tarball_size())) + key.set_contents_from_filename(NODE_MODULES_TARBALL) + print("node_modules cache successfully updated.") + _delete_file_quietly(NEED_TO_UPLOAD_MARKER) + else: + print("No need to upload anything.") + else: + raise SystemExit("Unrecognized mode {!r}".format(mode)) diff --git a/test-infra/requirements.txt b/test-infra/requirements.txt new file mode 100644 index 0000000000..95fbf1a3b0 --- /dev/null +++ b/test-infra/requirements.txt @@ -0,0 +1 @@ +boto==2.20.0 From 6b6b059ac388da516ed60e7b20098379466821b1 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Fri, 13 Dec 2013 17:20:45 -0800 Subject: [PATCH 2/2] time all the things --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2e71f553aa..78a89aabf6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,13 @@ language: node_js node_js: - 0.10 before_install: - - sudo pip install --use-mirrors -r ./test-infra/requirements.txt + - time sudo pip install --use-mirrors -r ./test-infra/requirements.txt install: - - gem install jekyll - - npm install -g grunt-cli - - ./test-infra/node_modules_cache.py download || npm install + - time gem install jekyll + - time npm install -g grunt-cli + - time ./test-infra/node_modules_cache.py download || time npm install after_script: - - ./test-infra/node_modules_cache.py upload + - time ./test-infra/node_modules_cache.py upload env: global: - SAUCE_USERNAME: bootstrap