diff --git a/Rakefile b/Rakefile
index b023c15..50cc3cf 100644
--- a/Rakefile
+++ b/Rakefile
@@ -2,6 +2,56 @@ require "rake/testtask"
 
 task :default => :test
 
-Rake::TestTask.new do |t|
-  t.warning = true
+$:.unshift File.expand_path("../lib", __FILE__)
+require "execjs/runtimes"
+
+tests = namespace :test do |tests|
+  ExecJS::Runtimes.names.each do |name|
+    task(name.downcase) do
+      ENV["EXECJS_RUNTIME"] = name
+    end
+
+    Rake::TestTask.new(name.downcase) do |t|
+      t.libs << "test"
+      t.warning = true
+    end
+  end
+end
+
+def banner(text)
+  warn ""
+  warn "=" * Rake.application.terminal_width
+  warn text
+  warn "=" * Rake.application.terminal_width
+  warn ""
+end
+
+desc "Run tests for all installed runtimes"
+task :test do
+  passed  = []
+  failed  = []
+  skipped = []
+
+  tests.tasks.each do |task|
+    banner "Running #{task.name}"
+
+    begin
+      task.invoke
+    rescue Exception => e
+      if e.message[/Command failed with status \((\d+)\)/, 1] == "2"
+        skipped << task.name
+      else
+        failed << task.name
+      end
+    else
+      passed << task.name
+    end
+  end
+
+  messages = ["PASSED: #{passed.join(", ")}"]
+  messages << "FAILURES: #{failed.join(", ")}" if failed.any?
+  messages << "SKIPPED: #{skipped.join(", ")}" if skipped.any?
+  banner messages.join("\n")
+
+  raise "test failures" if failed.any?
 end
diff --git a/test/execjs_test.rb b/test/execjs_test.rb
new file mode 100644
index 0000000..e17269a
--- /dev/null
+++ b/test/execjs_test.rb
@@ -0,0 +1,9 @@
+require "test/unit"
+require "execjs/module"
+
+begin
+  require "execjs"
+rescue ExecJS::RuntimeUnavailable => e
+  warn e
+  exit 2
+end
diff --git a/test/test_execjs.rb b/test/test_execjs.rb
index 0a67ff8..f7706f5 100644
--- a/test/test_execjs.rb
+++ b/test/test_execjs.rb
@@ -1,5 +1,4 @@
-require "execjs"
-require "test/unit"
+require "execjs_test"
 
 class TestExecJS < Test::Unit::TestCase
   def test_exec
diff --git a/test/test_runtimes.rb b/test/test_runtime.rb
similarity index 74%
rename from test/test_runtimes.rb
rename to test/test_runtime.rb
index 7a45d50..a983444 100644
--- a/test/test_runtimes.rb
+++ b/test/test_runtime.rb
@@ -1,8 +1,11 @@
 # -*- coding: utf-8 -*-
-require "execjs"
-require "test/unit"
+require "execjs_test"
+
+class TestRuntime < Test::Unit::TestCase
+  def setup
+    @runtime = ExecJS::Runtimes.autodetect
+  end
 
-module TestRuntime
   def test_exec
     assert_nil @runtime.exec("1")
     assert_nil @runtime.exec("return")
@@ -76,46 +79,3 @@ module TestRuntime
     end
   end
 end
-
-runtimes = [
-  "RubyRacer",
-  "RubyRhino",
-  "Mustang",
-  "Node",
-  "JavaScriptCore",
-  "Spidermonkey",
- "JScript"]
-
-warn "Runtime Support:"
-runtimes.each do |name|
-  runtime = ExecJS::Runtimes.const_get(name)
-  ok = runtime.available?
-
-  warn " %s %-21s %s" %
-    if ok
-      ["✓", runtime.name, "Found"]
-    else
-      [" ", runtime.name, "Not found"]
-    end
-
-  if ok
-    klass_name = "Test#{name}"
-    instance_eval "class ::#{klass_name} < Test::Unit::TestCase; end"
-    test_suite = Kernel.const_get(klass_name)
-
-    test_suite.instance_eval do
-      include TestRuntime
-
-      instance_exec do
-        define_method(:name) do
-          runtime.name
-        end
-      end
-
-      define_method(:setup) do
-        instance_variable_set(:@runtime, runtime)
-      end
-    end
-  end
-end
-warn ""