diff --git a/Project.toml b/Project.toml
index 8eb6e4a35..678827221 100644
--- a/Project.toml
+++ b/Project.toml
@@ -16,7 +16,7 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 ChainRulesCore = "0.9.16"
 ChainRulesTestUtils = "0.5"
 Compat = "3"
-FiniteDifferences = "0.11"
+FiniteDifferences = "0.11.4"
 Reexport = "0.2"
 Requires = "0.5.2, 1"
 julia = "1"
diff --git a/test/rulesets/Base/fastmath_able.jl b/test/rulesets/Base/fastmath_able.jl
index 989e65bc9..e83737a35 100644
--- a/test/rulesets/Base/fastmath_able.jl
+++ b/test/rulesets/Base/fastmath_able.jl
@@ -41,8 +41,6 @@ end
 # Do not add any tests here for functions that do not have varients in Base.FastMath
 # e.g. do not add `foo` unless `Base.FastMath.foo_fast` exists.
 const FASTABLE_AST = quote
-    is_fastmath_mode = sin === Base.FastMath.sin_fast
-
     @testset "Trig" begin
         @testset "Basics" for x = (Float64(π)-0.01, Complex(π, π/2))
             test_scalar(sin, x)
@@ -106,7 +104,7 @@ const FASTABLE_AST = quote
             complex_jacobian_test(angle, z)
         end
         @test frule((Zero(), randn()), angle, randn())[2] === Zero()
-        @test rrule(angle, randn())[2](randn())[2]        === Zero()
+        @test rrule(angle, randn())[2](randn())[2] === Zero()
 
         # test that real primal with complex tangent gives complex tangent
         ΔΩ = randn(ComplexF64)
@@ -122,7 +120,7 @@ const FASTABLE_AST = quote
         for x in (-4.1, 6.4, 0.0, 0.0 + 0.0im, 0.5 + 0.25im)
             test_scalar(+, x)
             test_scalar(-, x)
-            test_scalar(atan, x; rtol=(is_fastmath_mode ? 1e-7 : 1e-9))
+            test_scalar(atan, x)
         end
     end
 
@@ -141,9 +139,8 @@ const FASTABLE_AST = quote
             y, Δy, ȳ = rand(T, 3)
             Δz = randn(typeof(f(x, y)))
 
-            # some tests struggle in fast_math mode to get accurasy so we relax it some.
-            frule_test(f, (x, Δx), (y, Δy); rtol=(is_fastmath_mode ? 1e-5 : 1e-7))
-            rrule_test(f, Δz, (x, x̄), (y, ȳ); rtol=(is_fastmath_mode ? 1e-5 : 1e-7))
+            frule_test(f, (x, Δx), (y, Δy))
+            rrule_test(f, Δz, (x, x̄), (y, ȳ))
         end
 
         @testset "$f(x::$T, y::$T) type check" for f in (/, +, -,\, hypot, ^), T in (Float32, Float64)
diff --git a/test/rulesets/LinearAlgebra/blas.jl b/test/rulesets/LinearAlgebra/blas.jl
index 4c03d66e0..b61ec45c5 100644
--- a/test/rulesets/LinearAlgebra/blas.jl
+++ b/test/rulesets/LinearAlgebra/blas.jl
@@ -33,7 +33,7 @@
                 n = 10
                 x, ẋ, x̄ = randn(T, n), randn(T, n), randn(T, n)
                 frule_test(BLAS.nrm2, (x, ẋ))
-                rrule_test(BLAS.nrm2, randn(), (x, x̄); rtol=1e-7)
+                rrule_test(BLAS.nrm2, randn(), (x, x̄))
             end
         end
 
diff --git a/test/rulesets/LinearAlgebra/factorization.jl b/test/rulesets/LinearAlgebra/factorization.jl
index 24eba7088..60824a4f4 100644
--- a/test/rulesets/LinearAlgebra/factorization.jl
+++ b/test/rulesets/LinearAlgebra/factorization.jl
@@ -92,10 +92,10 @@ using ChainRules: level2partition, level3partition, chol_blocked_rev, chol_unblo
                 ΔF = unthunk(dF)
                 _, dX = dX_pullback(ΔF)
                 X̄_ad = dot(unthunk(dX), V)
-                X̄_fd = central_fdm(5, 1)(0.000_001) do ε
+                X̄_fd = _fdm(0.0) do ε
                     dot(Ȳ, getproperty(cholesky(X .+ ε .* V), p))
                 end
-                @test X̄_ad ≈ X̄_fd rtol=1e-4
+                @test X̄_ad ≈ X̄_fd rtol=1e-6 atol=1e-6
             end
         end
         @testset "helper functions" begin