diff --git a/tests/base.py b/tests/base.py
index 656defd02..427fbbc3d 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -3,8 +3,6 @@
 import random
 import string
 
-import os.path as osp
-
 from evalai.utils.config import AUTH_TOKEN_FILE_NAME, AUTH_TOKEN_DIR
 
 random.seed(10)
@@ -14,9 +12,10 @@ class BaseTestClass:
 
     @classmethod
     def setup_class(cls):
-        if not os.path.exists(AUTH_TOKEN_DIR):
+        token_file = os.path.join(AUTH_TOKEN_DIR, AUTH_TOKEN_FILE_NAME)
+        if not os.path.exists(token_file):
             os.makedirs(AUTH_TOKEN_DIR)
-            with open(osp.join(AUTH_TOKEN_DIR, AUTH_TOKEN_FILE_NAME), 'w') as fw:
+            with open(token_file, 'w') as fw:
                 token = ''.join(random.choice(string.ascii_lowercase) for _ in range(40))
                 data = {"token": "{}".format(token)}
                 fw.write(json.dumps(data))
diff --git a/tests/test_auth.py b/tests/test_auth.py
index adb36ee55..f421dadc1 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -1,7 +1,7 @@
 import json
 import os
 import responses
-import shutil
+
 from beautifultable import BeautifulTable
 from click.testing import CliRunner
 
@@ -10,6 +10,7 @@
 from evalai.utils.urls import URLS
 from evalai.utils.config import (API_HOST_URL,
                                  AUTH_TOKEN_DIR,
+                                 AUTH_TOKEN_FILE_NAME,
                                  HOST_URL_FILE_PATH)
 from evalai.utils.common import convert_UTC_date_to_local
 
@@ -19,9 +20,16 @@
 
 class TestGetUserAuthToken(BaseTestClass):
 
+    token_file = os.path.join(AUTH_TOKEN_DIR, AUTH_TOKEN_FILE_NAME)
+
     def setup(self):
-        if os.path.exists(AUTH_TOKEN_DIR):
-            shutil.rmtree(AUTH_TOKEN_DIR)
+        with open(self.token_file) as fo:
+            self.token = fo.read()
+        os.remove(self.token_file)
+
+    def teardown(self):
+        with open(self.token_file, 'w') as f:
+            f.write(self.token)
 
     def test_get_user_auth_token_when_file_does_not_exist(self):
         expected = ("\nThe authentication token json file doesn't exists at the required path. "