diff --git a/asks/errors.py b/asks/errors.py
index 75ab478..5572b76 100644
--- a/asks/errors.py
+++ b/asks/errors.py
@@ -11,6 +11,9 @@ class AsksException(Exception):
 
 
 class TooManyRedirects(AsksException):
+    def __init__(self, response):
+        super().__init__("Max redirects exceeded!")
+        self.response = response
     pass
 
 
diff --git a/asks/request_object.py b/asks/request_object.py
index 2f4a7ae..352d65a 100644
--- a/asks/request_object.py
+++ b/asks/request_object.py
@@ -269,7 +269,11 @@ async def _request_io(self, h11_request, h11_body, h11_connection):
         # check redirects
         if self.method != 'HEAD':
             if self.max_redirects < 0:
-                raise TooManyRedirects
+                # The most recent response is already on this history stack at this point
+                # Pop it and set it's history to the remaining stack.
+                response_obj = self.history_objects.pop()
+                response_obj.history = self.history_objects
+                raise TooManyRedirects(response_obj)
             response_obj = await self._redirect(response_obj)
         response_obj.history = self.history_objects
 
diff --git a/tests/test_anyio.py b/tests/test_anyio.py
index 8a7d0e9..a28697f 100644
--- a/tests/test_anyio.py
+++ b/tests/test_anyio.py
@@ -150,9 +150,11 @@ async def test_http_redirect(server):
 )
 @curio_run
 async def test_http_max_redirect_error(server):
-    with pytest.raises(TooManyRedirects):
-        await asks.get(server.http_test_url + "/redirect_max", max_redirects=1)
-
+    with pytest.raises(TooManyRedirects) as err:
+        r = await asks.get(server.http_test_url + "/redirect_max", max_redirects=1)
+    resp = err.value.response
+    assert resp.history[0].url == server.http_test_url + "/redirect_max"
+    assert len(err.value.response.history) == 1
 
 @Server(
     _TEST_LOC,