diff --git a/.travis.yml b/.travis.yml
index 3f727555bd7..1bc48b0b002 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,8 +13,13 @@ script:
  - rm -rf target/ Cargo.lock
  - cargo test --features debugging
 
- # only test regex_macros if it compiles
- - if [[ "$(cargo build --features 'debugging test-regex_macros')" = 101 ]]; then cargo test --features 'debugging test-regex_macros'; fi
+ - # only test regex_macros if it compiles
+ - |
+    #!/bin/bash
+    cargo test --no-run --features 'debugging test-regex_macros'
+    if [ "$?" = 101 ]; then
+        cargo test --features 'debugging test-regex_macros'
+    fi
 
 # trigger rebuild of the clippy-service, to keep it up to date with clippy itself
 after_success:
diff --git a/tests/compile-test.rs b/tests/compile-test.rs
index ff2d94d2777..6dcd1ff0524 100644
--- a/tests/compile-test.rs
+++ b/tests/compile-test.rs
@@ -3,7 +3,7 @@ extern crate compiletest_rs as compiletest;
 use std::path::PathBuf;
 use std::env::var;
 
-fn run_mode(mode: &'static str) {
+fn run_mode(dir: &'static str, mode: &'static str) {
     let mut config = compiletest::default_config();
 
     let cfg_mode = mode.parse().ok().expect("Invalid mode");
@@ -14,7 +14,7 @@ fn run_mode(mode: &'static str) {
     }
 
     config.mode = cfg_mode;
-    config.src_base = PathBuf::from(format!("tests/{}", mode));
+    config.src_base = PathBuf::from(format!("tests/{}", dir));
 
     compiletest::run_tests(&config);
 }
@@ -22,13 +22,13 @@ fn run_mode(mode: &'static str) {
 #[test]
 #[cfg(not(feature = "test-regex_macros"))]
 fn compile_test() {
-    run_mode("run-pass");
-    run_mode("compile-fail");
+    run_mode("run-pass", "run-pass");
+    run_mode("compile-fail", "compile-fail");
 }
 
 #[test]
 #[cfg(feature = "test-regex_macros")]
 fn compile_test() {
-    run_mode("run-pass-regex_macros");
-    run_mode("compile-fail-regex_macros");
+    run_mode("run-pass-regex_macros", "run-pass");
+    run_mode("compile-fail-regex_macros", "compile-fail");
 }