Merge pull request #11869 from twbs/cache-node_modules

use S3-based node_modules caching

.travis.yml

@@ -1,10 +1,18 @@
 language: node_js
 node_js:
   - 0.10
-before_script:
-  - gem install jekyll
-  - npm install -g grunt-cli
+before_install:
+  - time sudo pip install --use-mirrors -r ./test-infra/requirements.txt
+install:
+  - time gem install jekyll
+  - time npm install -g grunt-cli
+  - time ./test-infra/node_modules_cache.py download || time npm install
+after_script:
+  - time ./test-infra/node_modules_cache.py upload
 env:
   global:
   - SAUCE_USERNAME: bootstrap
   - secure: "pJkBwnuae9dKU5tEcCqccfS1QQw7/meEcfz63fM7ba7QJNjoA6BaXj08L5Z3Vb5vBmVPwBawxo5Hp0jC0r/Z/O0hGnAmz/Cz09L+cy7dSAZ9x4hvZePSja/UAusaB5ogMoO8l2b773MzgQeSmrLbExr9BWLeqEfjC2hFgdgHLaQ="
+  - secure: "gqjqISbxBJK6byFbsmr1AyP1qoWH+rap06A2gI7v72+Tn2PU2nYkIMUkCvhZw6K889jv+LhQ/ybcBxDOXHpNCExCnSgB4dcnmYp+9oeNZb37jSP0rQ+Ib4OTLjzc3/FawE/fUq5kukZTC7porzc/k0qJNLAZRx3YLALmK1GIdUY="
+  - secure: "Gghh/e3Gsbj1+4RR9Lh2aR/xJl35HWiHqlPIeSUqE9D7uDCVTAwNce/dGL3Ew7uJPfJ6Pgr70wD3zgu3stw0Zmzayax0hiDtGwcQCxVIER08wqGANK9C2Q7PYJkNTNtiTo6ehKWbdV4Z+/U+TEYyQfpQTDbAFYk/vVpsdjp0Lmc="
+  - secure: "RTbRdx4G/2OTLfrZtP1VbRljxEmd6A1F3GqXboeQTldsnAlwpsES65es5CE3ub/rmixLApOY9ot7OPmNixFgC2Y8xOsV7lNCC62QVpmqQEDyGFFQKb3yO6/dmwQxdsCqGfzf9Np6Wh5V22QFvr50ZLKLd7Uhd9oXMDIk/z1MJ3o="

test-infra/node_modules_cache.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python2.7
+from __future__ import absolute_import, unicode_literals, print_function, division
+
+from sys import argv
+from os import environ, stat, remove as _delete_file
+from os.path import isfile
+from hashlib import sha256
+from subprocess import check_call as run
+
+from boto.s3.connection import S3Connection
+from boto.s3.key import Key
+from boto.exception import S3ResponseError
+
+
+NODE_MODULES_TARBALL = 'node_modules.tar.gz'
+NEED_TO_UPLOAD_MARKER = '.need-to-upload'
+BYTES_PER_MB = 1024 * 1024
+try:
+    BUCKET_NAME = environ['TWBS_S3_BUCKET']
+except KeyError:
+    raise SystemExit("TWBS_S3_BUCKET environment variable not set!")
+
+
+def _sha256_of_file(filename):
+    hasher = sha256()
+    with open(filename, 'rb') as input_file:
+        hasher.update(input_file.read())
+    return hasher.hexdigest()
+
+
+def _delete_file_quietly(filename):
+    try:
+        _delete_file(filename)
+    except (OSError, IOError):
+        pass
+
+
+def _tarball_size():
+    kib = stat(NODE_MODULES_TARBALL).st_size // BYTES_PER_MB
+    return "{} MiB".format(kib)
+
+
+if __name__ == '__main__':
+    # Uses environment variables:
+    #   AWS_ACCESS_KEY_ID - AWS Access Key ID
+    #   AWS_SECRET_ACCESS_KEY - AWS Secret Access Key
+    argv.pop(0)
+    if len(argv) != 1:
+        raise SystemExit("USAGE: node_modules_cache.py <download | upload>")
+    mode = argv.pop()
+
+    conn = S3Connection()
+    bucket = conn.lookup(BUCKET_NAME)
+    if bucket is None:
+        raise SystemExit("Could not access bucket!")
+
+    package_json_hash = _sha256_of_file('package.json')
+    print('sha256(package.json) = ' + package_json_hash)
+
+    key = Key(bucket, package_json_hash)
+    key.storage_class = 'REDUCED_REDUNDANCY'
+
+    if mode == 'download':
+        _delete_file_quietly(NEED_TO_UPLOAD_MARKER)
+        try:
+            print("Downloading tarball from S3...")
+            key.get_contents_to_filename(NODE_MODULES_TARBALL)
+        except S3ResponseError as err:
+            open(NEED_TO_UPLOAD_MARKER, 'a').close()
+            print(err)
+            raise SystemExit("Cached node_modules download failed!")
+        print("Downloaded {}.".format(_tarball_size()))
+        print("Extracting tarball...")
+        run(['tar', 'xzf', NODE_MODULES_TARBALL])
+        print("node_modules successfully installed from cache.")
+    elif mode == 'upload':
+        if isfile(NEED_TO_UPLOAD_MARKER):
+            print("Creating tarball...")
+            run(['tar', 'czf', NODE_MODULES_TARBALL, 'node_modules'])
+            print("Uploading tarball to S3... ({})".format(_tarball_size()))
+            key.set_contents_from_filename(NODE_MODULES_TARBALL)
+            print("node_modules cache successfully updated.")
+            _delete_file_quietly(NEED_TO_UPLOAD_MARKER)
+        else:
+            print("No need to upload anything.")
+    else:
+        raise SystemExit("Unrecognized mode {!r}".format(mode))

test-infra/requirements.txt

@@ -0,0 +1 @@
+boto==2.20.0