gh-112730: Make the test suite resilient to color-activation environment variables (#117672)

Author: pablogsal

.github/workflows/reusable-ubuntu.yml

@@ -14,6 +14,7 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-20.04
     env:
+      FORCE_COLOR: 1
       OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:

Lib/doctest.py

@@ -1556,7 +1556,11 @@ def out(s):
         # Make sure sys.displayhook just prints the value to stdout
         save_displayhook = sys.displayhook
         sys.displayhook = sys.__displayhook__
-
+        saved_can_colorize = traceback._can_colorize
+        traceback._can_colorize = lambda: False
+        color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None}
+        for key in color_variables:
+            color_variables[key] = os.environ.pop(key, None)
         try:
             return self.__run(test, compileflags, out)
         finally:
@@ -1565,6 +1569,10 @@ def out(s):
             sys.settrace(save_trace)
             linecache.getlines = self.save_linecache_getlines
             sys.displayhook = save_displayhook
+            traceback._can_colorize = saved_can_colorize
+            for key, value in color_variables.items():
+                if value is not None:
+                    os.environ[key] = value
             if clear_globs:
                 test.globs.clear()
                 import builtins

Lib/idlelib/idle_test/test_run.py

@@ -8,6 +8,7 @@
 from unittest import mock
 import idlelib
 from idlelib.idle_test.mock_idle import Func
+from test.support import force_not_colorized
 
 idlelib.testing = True  # Use {} for executing test user code.
 
@@ -46,6 +47,7 @@ def __eq__(self, other):
                  "Did you mean: 'real'?\n"),
             )
 
+    @force_not_colorized
     def test_get_message(self):
         for code, exc, msg in self.data:
             with self.subTest(code=code):
@@ -57,6 +59,7 @@ def test_get_message(self):
                     expect = f'{exc.__name__}: {msg}'
                     self.assertEqual(actual, expect)
 
+    @force_not_colorized
     @mock.patch.object(run, 'cleanup_traceback',
                        new_callable=lambda: (lambda t, e: None))
     def test_get_multiple_message(self, mock):

Lib/test/support/__init__.py

@@ -59,6 +59,7 @@
     "Py_DEBUG", "exceeds_recursion_limit", "get_c_recursion_limit",
     "skip_on_s390x",
     "without_optimizer",
+    "force_not_colorized"
     ]
 
 
@@ -2557,3 +2558,22 @@ def copy_python_src_ignore(path, names):
             'build',
         }
     return ignored
+
+def force_not_colorized(func):
+    """Force the terminal not to be colorized."""
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        import traceback
+        original_fn = traceback._can_colorize
+        variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None}
+        try:
+            for key in variables:
+                variables[key] = os.environ.pop(key, None)
+            traceback._can_colorize = lambda: False
+            return func(*args, **kwargs)
+        finally:
+            traceback._can_colorize = original_fn
+            for key, value in variables.items():
+                if value is not None:
+                    os.environ[key] = value
+    return wrapper

Lib/test/test_cmd_line.py

@@ -10,6 +10,7 @@
 import unittest
 from test import support
 from test.support import os_helper
+from test.support import force_not_colorized
 from test.support.script_helper import (
     spawn_python, kill_python, assert_python_ok, assert_python_failure,
     interpreter_requires_environment
@@ -1027,6 +1028,7 @@ def test_sys_flags_not_set(self):
 
 
 class SyntaxErrorTests(unittest.TestCase):
+    @force_not_colorized
     def check_string(self, code):
         proc = subprocess.run([sys.executable, "-"], input=code,
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)

Lib/test/test_exceptions.py

@@ -12,7 +12,8 @@
 from test.support import (captured_stderr, check_impl_detail,
                           cpython_only, gc_collect,
                           no_tracing, script_helper,
-                          SuppressCrashReport)
+                          SuppressCrashReport,
+                          force_not_colorized)
 from test.support.import_helper import import_module
 from test.support.os_helper import TESTFN, unlink
 from test.support.warnings_helper import check_warnings
@@ -41,6 +42,7 @@ def __str__(self):
 
 # XXX This is not really enough, each *operation* should be tested!
 
+
 class ExceptionTests(unittest.TestCase):
 
     def raise_catch(self, exc, excname):
@@ -1994,6 +1996,7 @@ def write_source(self, source):
         _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN)
         return err.decode('utf-8').splitlines()
 
+    @force_not_colorized
     def test_assertion_error_location(self):
         cases = [
             ('assert None',
@@ -2070,6 +2073,7 @@ def test_assertion_error_location(self):
                 result = self.write_source(source)
                 self.assertEqual(result[-3:], expected)
 
+    @force_not_colorized
     def test_multiline_not_highlighted(self):
         cases = [
             ("""
@@ -2102,6 +2106,7 @@ def test_multiline_not_highlighted(self):
 
 
 class SyntaxErrorTests(unittest.TestCase):
+    @force_not_colorized
     def test_range_of_offsets(self):
         cases = [
             # Basic range from 2->7

Lib/test/test_interpreters/test_api.py

@@ -12,6 +12,7 @@
 _interpreters = import_helper.import_module('_interpreters')
 from test.support import Py_GIL_DISABLED
 from test.support import interpreters
+from test.support import force_not_colorized
 from test.support.interpreters import (
     InterpreterError, InterpreterNotFoundError, ExecutionFailed,
 )
@@ -735,6 +736,7 @@ def test_failure(self):
         with self.assertRaises(ExecutionFailed):
             interp.exec('raise Exception')
 
+    @force_not_colorized
     def test_display_preserved_exception(self):
         tempdir = self.temp_dir()
         modfile = self.make_module('spam', tempdir, text="""

Lib/test/test_sys.py

@@ -16,6 +16,7 @@
 from test.support.script_helper import assert_python_ok, assert_python_failure
 from test.support import threading_helper
 from test.support import import_helper
+from test.support import force_not_colorized
 try:
     from test.support import interpreters
 except ImportError:
@@ -145,6 +146,7 @@ def f():
 
 class ExceptHookTest(unittest.TestCase):
 
+    @force_not_colorized
     def test_original_excepthook(self):
         try:
             raise ValueError(42)
@@ -156,6 +158,7 @@ def test_original_excepthook(self):
 
         self.assertRaises(TypeError, sys.__excepthook__)
 
+    @force_not_colorized
     def test_excepthook_bytes_filename(self):
         # bpo-37467: sys.excepthook() must not crash if a filename
         # is a bytes string
@@ -793,6 +796,7 @@ def test_sys_getwindowsversion_no_instantiation(self):
     def test_clear_type_cache(self):
         sys._clear_type_cache()
 
+    @force_not_colorized
     @support.requires_subprocess()
     def test_ioencoding(self):
         env = dict(os.environ)
@@ -1108,6 +1112,7 @@ def test_getandroidapilevel(self):
         self.assertIsInstance(level, int)
         self.assertGreater(level, 0)
 
+    @force_not_colorized
     @support.requires_subprocess()
     def test_sys_tracebacklimit(self):
         code = """if 1:

Lib/test/test_threading.py

@@ -7,6 +7,7 @@
 from test.support import verbose, cpython_only, os_helper
 from test.support.import_helper import import_module
 from test.support.script_helper import assert_python_ok, assert_python_failure
+from test.support import force_not_colorized
 
 import random
 import sys
@@ -1793,6 +1794,7 @@ def setUp(self):
         restore_default_excepthook(self)
         super().setUp()
 
+    @force_not_colorized
     def test_excepthook(self):
         with support.captured_output("stderr") as stderr:
             thread = ThreadRunFail(name="excepthook thread")
@@ -1806,6 +1808,7 @@ def test_excepthook(self):
         self.assertIn('ValueError: run failed', stderr)
 
     @support.cpython_only
+    @force_not_colorized
     def test_excepthook_thread_None(self):
         # threading.excepthook called with thread=None: log the thread
         # identifier in this case.

Lib/test/test_traceback.py

@@ -21,6 +21,7 @@
 from test.support.os_helper import TESTFN, unlink
 from test.support.script_helper import assert_python_ok, assert_python_failure
 from test.support.import_helper import forget
+from test.support import force_not_colorized
 
 import json
 import textwrap
@@ -39,6 +40,13 @@
 
 LEVENSHTEIN_DATA_FILE = Path(__file__).parent / 'levenshtein_examples.json'
 
+ORIGINAL_CAN_COLORIZE = traceback._can_colorize
+
+def setUpModule():
+    traceback._can_colorize = lambda: False
+
+def tearDownModule():
+    traceback._can_colorize = ORIGINAL_CAN_COLORIZE
 
 class TracebackCases(unittest.TestCase):
     # For now, a very minimal set of tests.  I want to be sure that
@@ -124,6 +132,7 @@ def test_nocaret(self):
         self.assertEqual(len(err), 3)
         self.assertEqual(err[1].strip(), "bad syntax")
 
+    @force_not_colorized
     def test_no_caret_with_no_debug_ranges_flag(self):
         # Make sure that if `-X no_debug_ranges` is used, there are no carets
         # in the traceback.
@@ -401,7 +410,7 @@ def do_test(firstlines, message, charset, lineno):
                         """.format(firstlines, message))
 
                 process = subprocess.Popen([sys.executable, TESTFN],
-                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env={})
                 stdout, stderr = process.communicate()
                 stdout = stdout.decode(output_encoding).splitlines()
             finally:
@@ -4354,13 +4363,18 @@ def foo():
             f'{boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}']
         self.assertEqual(actual, expected)
 
+    @force_not_colorized
     def test_colorized_detection_checks_for_environment_variables(self):
         if sys.platform == "win32":
             virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", return_value=True)
         else:
             virtual_patching = contextlib.nullcontext()
         with virtual_patching:
-            with unittest.mock.patch("os.isatty") as isatty_mock:
+
+            flags = unittest.mock.MagicMock(ignore_environment=False)
+            with (unittest.mock.patch("os.isatty") as isatty_mock,
+                  unittest.mock.patch("sys.flags", flags),
+                  unittest.mock.patch("traceback._can_colorize", ORIGINAL_CAN_COLORIZE)):
                 isatty_mock.return_value = True
                 with unittest.mock.patch("os.environ", {'TERM': 'dumb'}):
                     self.assertEqual(traceback._can_colorize(), False)
@@ -4379,7 +4393,8 @@ def test_colorized_detection_checks_for_environment_variables(self):
                 with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}):
                     self.assertEqual(traceback._can_colorize(), False)
                 isatty_mock.return_value = False
-                self.assertEqual(traceback._can_colorize(), False)
+                with unittest.mock.patch("os.environ", {}):
+                    self.assertEqual(traceback._can_colorize(), False)
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_tracemalloc.py

@@ -942,7 +942,7 @@ def check_env_var_invalid(self, nframe):
         with support.SuppressCrashReport():
             ok, stdout, stderr = assert_python_failure(
                 '-c', 'pass',
-                PYTHONTRACEMALLOC=str(nframe))
+                PYTHONTRACEMALLOC=str(nframe), __cleanenv=True)
 
         if b'ValueError: the number of frames must be in range' in stderr:
             return

Lib/test/test_warnings/__init__.py

@@ -12,6 +12,7 @@
 from test.support import import_helper
 from test.support import os_helper
 from test.support import warnings_helper
+from test.support import force_not_colorized
 from test.support.script_helper import assert_python_ok, assert_python_failure
 
 from test.test_warnings.data import package_helper
@@ -1239,6 +1240,7 @@ def test_comma_separated_warnings(self):
         self.assertEqual(stdout,
             b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']")
 
+    @force_not_colorized
     def test_envvar_and_command_line(self):
         rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c",
             "import sys; sys.stdout.write(str(sys.warnoptions))",
@@ -1247,6 +1249,7 @@ def test_envvar_and_command_line(self):
         self.assertEqual(stdout,
             b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']")
 
+    @force_not_colorized
     def test_conflicting_envvar_and_command_line(self):
         rc, stdout, stderr = assert_python_failure("-Werror::DeprecationWarning", "-c",
             "import sys, warnings; sys.stdout.write(str(sys.warnoptions)); "

Lib/traceback.py

@@ -141,24 +141,30 @@ def _can_colorize():
                 return False
         except (ImportError, AttributeError):
             return False
-
-    if os.environ.get("PYTHON_COLORS") == "0":
-        return False
-    if os.environ.get("PYTHON_COLORS") == "1":
-        return True
-    if "NO_COLOR" in os.environ:
-        return False
+    if not sys.flags.ignore_environment:
+        if os.environ.get("PYTHON_COLORS") == "0":
+            return False
+        if os.environ.get("PYTHON_COLORS") == "1":
+            return True
+        if "NO_COLOR" in os.environ:
+            return False
     if not _COLORIZE:
         return False
-    if "FORCE_COLOR" in os.environ:
-        return True
-    if os.environ.get("TERM") == "dumb":
+    if not sys.flags.ignore_environment:
+        if "FORCE_COLOR" in os.environ:
+            return True
+        if os.environ.get("TERM") == "dumb":
+            return False
+
+    if not hasattr(sys.stderr, "fileno"):
         return False
+
     try:
         return os.isatty(sys.stderr.fileno())
     except io.UnsupportedOperation:
         return sys.stderr.isatty()
 
+
 def _print_exception_bltin(exc, /):
     file = sys.stderr if sys.stderr is not None else sys.__stderr__
     colorize = _can_colorize()