Bug 1894157 - Remove Python2 compatibility code in Mach r=firefox-build-system-reviewers,glandium

Mach can currently only run on Python version 3.8 or higher, so it
doesn't make sense to continue having dead code that provides support
for Python2.

Differential Revision: https://phabricator.services.mozilla.com/D209030

build/mach_initialize.py

@@ -6,19 +6,9 @@ import math
 import os
 import shutil
 import sys
+from importlib.abc import MetaPathFinder
 from pathlib import Path
 
-if sys.version_info[0] < 3:
-    import __builtin__ as builtins
-
-    class MetaPathFinder(object):
-        pass
-
-else:
-    from importlib.abc import MetaPathFinder
-
-from types import ModuleType
-
 STATE_DIR_FIRST_RUN = """
 Mach and the build system store shared state in a common directory
 on the filesystem. The following directory will be created:
@@ -485,76 +475,6 @@ def _create_state_dir():
     return state_dir
 
 
-# Hook import such that .pyc/.pyo files without a corresponding .py file in
-# the source directory are essentially ignored. See further below for details
-# and caveats.
-# Objdirs outside the source directory are ignored because in most cases, if
-# a .pyc/.pyo file exists there, a .py file will be next to it anyways.
-class ImportHook(object):
-    def __init__(self, original_import):
-        self._original_import = original_import
-        # Assume the source directory is the parent directory of the one
-        # containing this file.
-        self._source_dir = (
-            os.path.normcase(
-                os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-            )
-            + os.sep
-        )
-        self._modules = set()
-
-    def __call__(self, name, globals=None, locals=None, fromlist=None, level=-1):
-        if sys.version_info[0] >= 3 and level < 0:
-            level = 0
-
-        # name might be a relative import. Instead of figuring out what that
-        # resolves to, which is complex, just rely on the real import.
-        # Since we don't know the full module name, we can't check sys.modules,
-        # so we need to keep track of which modules we've already seen to avoid
-        # to stat() them again when they are imported multiple times.
-        module = self._original_import(name, globals, locals, fromlist, level)
-
-        # Some tests replace modules in sys.modules with non-module instances.
-        if not isinstance(module, ModuleType):
-            return module
-
-        resolved_name = module.__name__
-        if resolved_name in self._modules:
-            return module
-        self._modules.add(resolved_name)
-
-        # Builtin modules don't have a __file__ attribute.
-        if not getattr(module, "__file__", None):
-            return module
-
-        # Note: module.__file__ is not always absolute.
-        path = os.path.normcase(os.path.abspath(module.__file__))
-        # Note: we could avoid normcase and abspath above for non pyc/pyo
-        # files, but those are actually rare, so it doesn't really matter.
-        if not path.endswith((".pyc", ".pyo")):
-            return module
-
-        # Ignore modules outside our source directory
-        if not path.startswith(self._source_dir):
-            return module
-
-        # If there is no .py corresponding to the .pyc/.pyo module we're
-        # loading, remove the .pyc/.pyo file, and reload the module.
-        # Since we already loaded the .pyc/.pyo module, if it had side
-        # effects, they will have happened already, and loading the module
-        # with the same name, from another directory may have the same side
-        # effects (or different ones). We assume it's not a problem for the
-        # python modules under our source directory (either because it
-        # doesn't happen or because it doesn't matter).
-        if not os.path.exists(module.__file__[:-1]):
-            if os.path.exists(module.__file__):
-                os.remove(module.__file__)
-            del sys.modules[module.__name__]
-            module = self(name, globals, locals, fromlist, level)
-
-        return module
-
-
 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 # the source directory are essentially ignored. See further below for details
 # and caveats.
@@ -616,8 +536,4 @@ def hook(finder):
     return finder
 
 
-# Install our hook. This can be deleted when the Python 3 migration is complete.
+sys.meta_path = [hook(c) for c in sys.meta_path]
-if sys.version_info[0] < 3:
-    builtins.__import__ = ImportHook(builtins.__import__)
-else:
-    sys.meta_path = [hook(c) for c in sys.meta_path]

python/mach/mach/main.py

@@ -6,7 +6,6 @@
 # (mach). It is packaged as a module because everything is a library.
 
 import argparse
-import codecs
 import logging
 import os
 import sys
@@ -258,16 +257,6 @@ To see more help for a specific command, run:
         try:
             self.load_settings()
 
-            if sys.version_info < (3, 0):
-                if stdin.encoding is None:
-                    sys.stdin = codecs.getreader("utf-8")(stdin)
-
-                if stdout.encoding is None:
-                    sys.stdout = codecs.getwriter("utf-8")(stdout)
-
-                if stderr.encoding is None:
-                    sys.stderr = codecs.getwriter("utf-8")(stderr)
-
             # Allow invoked processes (which may not have a handle on the
             # original stdout file descriptor) to know if the original stdout
             # is a TTY. This provides a mechanism to allow said processes to

python/mach/mach/telemetry.py

@@ -29,11 +29,10 @@ SITE_DIR = (Path(__file__) / ".." / ".." / ".." / "sites").resolve()
 def create_telemetry_from_environment(settings):
     """Creates and a Telemetry instance based on system details.
 
-    If telemetry isn't enabled, the current interpreter isn't Python 3, or Glean
+    If telemetry isn't enabled or Glean can't be imported, then a "mock" telemetry
-    can't be imported, then a "mock" telemetry instance is returned that doesn't
+    instance is returned that doesn't set or record any data. This allows consumers
-    set or record any data. This allows consumers to optimistically set telemetry
+    to optimistically set telemetry data without needing to specifically handle the
-    data without needing to specifically handle the case where the current system
+    case where the current system doesn't support it.
-    doesn't support it.
     """
 
     active_metadata = MozSiteMetadata.from_runtime()
@@ -42,8 +41,6 @@ def create_telemetry_from_environment(settings):
 
     if not (
         is_applicable_telemetry_environment()
-        # Glean is not compatible with Python 2
-        and sys.version_info >= (3, 0)
         # If not using a mach virtualenv (e.g.: bootstrap uses native python)
         # then we can't guarantee that the glean package that we import is a
         # compatible version. Therefore, don't use glean.

python/mach/mach/util.py

@@ -20,20 +20,12 @@ def setenv(key, value):
     """Compatibility shim to ensure the proper string type is used with
     os.environ for the version of Python being used.
     """
-    from six import text_type
-
     encoding = "mbcs" if sys.platform == "win32" else "utf-8"
 
-    if sys.version_info[0] == 2:
+    if isinstance(key, bytes):
-        if isinstance(key, text_type):
+        key = key.decode(encoding)
-            key = key.encode(encoding)
+    if isinstance(value, bytes):
-        if isinstance(value, text_type):
+        value = value.decode(encoding)
-            value = value.encode(encoding)
-    else:
-        if isinstance(key, bytes):
-            key = key.decode(encoding)
-        if isinstance(value, bytes):
-            value = value.decode(encoding)
 
     os.environ[key] = value
 

python/mozbuild/mozbuild/action/dump_env.py

@@ -12,19 +12,5 @@ sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 
 from shellutil import quote
 
-
+for key, value in os.environ.items():
-def environ():
-    # We would use six.ensure_text but the global Python isn't guaranteed to have
-    # the correct version of six installed.
-    def ensure_text(s):
-        if sys.version_info > (3, 0) or isinstance(s, unicode):
-            # os.environ always returns string keys and values in Python 3.
-            return s
-        else:
-            return s.decode("utf-8")
-
-    return [(ensure_text(k), ensure_text(v)) for (k, v) in os.environ.items()]
-
-
-for key, value in environ():
     print("%s=%s" % (key, quote(value)))

python/mozbuild/mozbuild/action/tooltool.py

@@ -61,25 +61,12 @@ REQUEST_HEADER_ATTRIBUTE_CHARS = re.compile(
 DEFAULT_MANIFEST_NAME = "manifest.tt"
 TOOLTOOL_PACKAGE_SUFFIX = ".TOOLTOOL-PACKAGE"
 HAWK_VER = 1
-PY3 = sys.version_info[0] == 3
-
-if PY3:
-    six_binary_type = bytes
-    unicode = (
-        str  # Silence `pyflakes` from reporting `undefined name 'unicode'` in Python 3.
-    )
-    import urllib.request as urllib2
-    from http.client import HTTPConnection, HTTPSConnection
-    from urllib.error import HTTPError, URLError
-    from urllib.parse import urljoin, urlparse
-    from urllib.request import Request
-else:
-    six_binary_type = str
-    import urllib2
-    from httplib import HTTPConnection, HTTPSConnection
-    from urllib2 import HTTPError, Request, URLError
-    from urlparse import urljoin, urlparse
 
+import urllib.request as urllib2
+from http.client import HTTPConnection, HTTPSConnection
+from urllib.error import HTTPError, URLError
+from urllib.parse import urljoin, urlparse
+from urllib.request import Request
 
 log = logging.getLogger(__name__)
 
@@ -205,9 +192,7 @@ def retriable(*retry_args, **retry_kwargs):
 
 
 def request_has_data(req):
-    if PY3:
+    return req.data is not None
-        return req.data is not None
-    return req.has_data()
 
 
 def get_hexdigest(val):
@@ -281,7 +266,7 @@ def random_string(length):
 
 
 def prepare_header_val(val):
-    if isinstance(val, six_binary_type):
+    if isinstance(val, bytes):
         val = val.decode("utf-8")
 
     if not REQUEST_HEADER_ATTRIBUTE_CHARS.match(val):
@@ -303,7 +288,7 @@ def parse_content_type(content_type):  # pragma: no cover
 
 def calculate_payload_hash(algorithm, payload, content_type):  # pragma: no cover
     parts = [
-        part if isinstance(part, six_binary_type) else part.encode("utf8")
+        part if isinstance(part, bytes) else part.encode("utf8")
         for part in [
             "hawk." + str(HAWK_VER) + ".payload\n",
             parse_content_type(content_type) + "\n",
@@ -337,7 +322,7 @@ def validate_taskcluster_credentials(credentials):
 
 
 def normalize_header_attr(val):
-    if isinstance(val, six_binary_type):
+    if isinstance(val, bytes):
         return val.decode("utf-8")
     return val  # pragma: no cover
 
@@ -390,10 +375,10 @@ def calculate_mac(
     log.debug("normalized resource for mac calc: {norm}".format(norm=normalized))
     digestmod = getattr(hashlib, algorithm)
 
-    if not isinstance(normalized, six_binary_type):
+    if not isinstance(normalized, bytes):
         normalized = normalized.encode("utf8")
 
-    if not isinstance(access_token, six_binary_type):
+    if not isinstance(access_token, bytes):
         access_token = access_token.encode("ascii")
 
     result = hmac.new(access_token, normalized, digestmod)
@@ -412,10 +397,7 @@ def make_taskcluster_header(credentials, req):
 
     content_hash = None
     if request_has_data(req):
-        if PY3:
+        data = req.data
-            data = req.data
-        else:
-            data = req.get_data()
         content_hash = calculate_payload_hash(  # pragma: no cover
             algorithm,
             data,
@@ -760,7 +742,7 @@ def open_manifest(manifest_file):
     """I know how to take a filename and load it into a Manifest object"""
     if os.path.exists(manifest_file):
         manifest = Manifest()
-        with open(manifest_file, "r" if PY3 else "rb") as f:
+        with open(manifest_file, "r") as f:
             manifest.load(f)
             log.debug("loaded manifest from file '%s'" % manifest_file)
         return manifest
@@ -865,12 +847,10 @@ def add_files(manifest_file, algorithm, filenames, version, visibility, unpack):
     for old_fr in old_manifest.file_records:
         if old_fr.filename not in new_filenames:
             new_manifest.file_records.append(old_fr)
-    if PY3:
+
-        with open(manifest_file, mode="w") as output:
+    with open(manifest_file, mode="w") as output:
-            new_manifest.dump(output, fmt="json")
+        new_manifest.dump(output, fmt="json")
-    else:
+
-        with open(manifest_file, mode="wb") as output:
-            new_manifest.dump(output, fmt="json")
     return all_files_added
 
 
@@ -1288,9 +1268,7 @@ def _send_batch(base_url, auth_file, batch, region):
     url = urljoin(base_url, "upload")
     if region is not None:
         url += "?region=" + region
-    data = json.dumps(batch)
+    data = json.dumps(batch).encode("utf-8")
-    if PY3:
-        data = data.encode("utf-8")
     req = Request(url, data, {"Content-Type": "application/json"})
     _authorize(req, auth_file)
     try:

python/mozbuild/mozbuild/backend/visualstudio.py

@@ -8,7 +8,6 @@
 import errno
 import os
 import re
-import sys
 import uuid
 from pathlib import Path
 from xml.dom import getDOMImplementation
@@ -35,8 +34,6 @@ MSNATVIS_NAMESPACE = "http://schemas.microsoft.com/vstudio/debugger/natvis/2010"
 
 
 def get_id(name):
-    if sys.version_info[0] == 2:
-        name = name.encode("utf-8")
     return str(uuid.uuid5(uuid.NAMESPACE_URL, name)).upper()
 
 

python/mozbuild/mozbuild/configure/lint.py

@@ -4,7 +4,6 @@
 
 import inspect
 import re
-import sys
 import types
 from dis import Bytecode
 from functools import wraps
@@ -25,27 +24,9 @@ from .help import HelpFormatter
 
 
 def code_replace(code, co_filename, co_name, co_firstlineno):
-    if sys.version_info < (3, 8):
+    return code.replace(
-        codetype_args = [
+        co_filename=co_filename, co_name=co_name, co_firstlineno=co_firstlineno
-            code.co_argcount,
+    )
-            code.co_kwonlyargcount,
-            code.co_nlocals,
-            code.co_stacksize,
-            code.co_flags,
-            code.co_code,
-            code.co_consts,
-            code.co_names,
-            code.co_varnames,
-            co_filename,
-            co_name,
-            co_firstlineno,
-            code.co_lnotab,
-        ]
-        return types.CodeType(*codetype_args)
-    else:
-        return code.replace(
-            co_filename=co_filename, co_name=co_name, co_firstlineno=co_firstlineno
-        )
 
 
 class LintSandbox(ConfigureSandbox):

python/mozbuild/mozbuild/doctor.py

@@ -3,7 +3,6 @@
 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import enum
-import locale
 import os
 import socket
 import subprocess
@@ -459,39 +458,6 @@ def mozillabuild(**kwargs) -> DoctorCheck:
     return DoctorCheck(name="mozillabuild", status=status, display_text=[desc])
 
 
-@check
-def bad_locale_utf8(**kwargs) -> DoctorCheck:
-    """Check to detect the invalid locale `UTF-8` on pre-3.8 Python."""
-    if sys.version_info >= (3, 8):
-        return DoctorCheck(
-            name="utf8 locale",
-            status=CheckStatus.SKIPPED,
-            display_text=["Python version has fixed utf-8 locale bug."],
-        )
-
-    try:
-        # This line will attempt to get and parse the locale.
-        locale.getdefaultlocale()
-
-        return DoctorCheck(
-            name="utf8 locale",
-            status=CheckStatus.OK,
-            display_text=["Python's locale is set to a valid value."],
-        )
-    except ValueError:
-        return DoctorCheck(
-            name="utf8 locale",
-            status=CheckStatus.FATAL,
-            display_text=[
-                "Your Python is using an invalid value for its locale.",
-                "Either update Python to version 3.8+, or set the following variables in ",
-                "your environment:",
-                "  export LC_ALL=en_US.UTF-8",
-                "  export LANG=en_US.UTF-8",
-            ],
-        )
-
-
 @check
 def artifact_build(
     topsrcdir: str, configure_args: Optional[List[str]], **kwargs

python/mozbuild/mozbuild/dotproperties.py

@@ -6,15 +6,9 @@
 
 import codecs
 import re
-import sys
 
 import six
 
-if sys.version_info[0] == 3:
-    str_type = str
-else:
-    str_type = basestring
-
 
 class DotProperties:
     r"""A thin representation of a key=value .properties file."""
@@ -29,7 +23,7 @@ class DotProperties:
 
         Ignores empty lines and comment lines."""
 
-        if isinstance(file, str_type):
+        if isinstance(file, str):
             f = codecs.open(file, "r", "utf-8")
         else:
             f = file

python/mozbuild/mozbuild/jar.py

@@ -630,10 +630,7 @@ def main(args=None):
     noise = logging.INFO
     if options.verbose is not None:
         noise = options.verbose and logging.DEBUG or logging.WARN
-    if sys.version_info[:2] > (2, 3):
+    logging.basicConfig(format="%(message)s")
-        logging.basicConfig(format="%(message)s")
-    else:
-        logging.basicConfig()
     logging.getLogger().setLevel(noise)
     topsrc = options.t
     topsrc = os.path.normpath(os.path.abspath(topsrc))

python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py

@@ -219,14 +219,10 @@ import inspect
 
 
 def node_to_name(code, node):
-    if (
+    if FORCE_DOWNGRADE_BEHAVIOR:
-        not FORCE_DOWNGRADE_BEHAVIOR
+        return node.__class__.__name__
-        and sys.version_info[0] >= 3
-        and sys.version_info[1] >= 8
-    ):
-        return ast.get_source_segment(code, node)
 
-    return node.__class__.__name__
+    return ast.get_source_segment(code, node)
 
 
 def get_attribute_label(node):
@@ -254,11 +250,7 @@ def get_attribute_label(node):
 def ast_get_source_segment(code, node):
     caller = inspect.stack()[1]
 
-    if "sphinx" in caller.filename or (
+    if "sphinx" in caller.filename or not FORCE_DOWNGRADE_BEHAVIOR:
-        not FORCE_DOWNGRADE_BEHAVIOR
-        and sys.version_info[0] >= 3
-        and sys.version_info[1] >= 8
-    ):
         return ast.original_get_source_segment(code, node)
 
     if caller.function == "assignment_node_to_source_filename_list":
@@ -271,9 +263,8 @@ def ast_get_source_segment(code, node):
 
 
 # Overwrite it so we don't accidently use it
-if sys.version_info[0] >= 3 and sys.version_info[1] >= 8:
+ast.original_get_source_segment = ast.get_source_segment
-    ast.original_get_source_segment = ast.get_source_segment
+ast.get_source_segment = ast_get_source_segment
-    ast.get_source_segment = ast_get_source_segment
 
 
 ##############################################