Bug 1466211 - Switch all |mach python-test| tests to run using pipenv; r=ahal

MozReview-Commit-ID: AzmdDgAgZgI

--HG--
extra : rebase_source : 8db60e196667ae97e0f994930c9adbb4d2030d34
extra : source : e550be515be30148c0a89c33953dff4148a6a171

Pipfile

@@ -16,7 +16,3 @@ requests = "==2.9.1"
 six = "==1.10.0"
 virtualenv = "==15.2.0"
 voluptuous = "==0.10.5"
-
-
-[requires]
-python_version = "2.7"

Pipfile.lock

@@ -1,12 +1,10 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "af4e239c88ce3d74e2e3dd7d352c3e8a203ce476c7369b2a4dc0eea7114996ba"
+            "sha256": "eb8b0a9771d4420f83fbbabf9952dc783aeefe9be455559de2f3ebff27caa93f"
         },
         "pipfile-spec": 6,
-        "requires": {
+        "requires": {},
-            "python_version": "2.7"
-        },
         "sources": [
             {
                 "name": "pypi",

python/Pipfile

@@ -1,36 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-"d5b4a14" = {path = "./mach"}
-"8ddb376" = {path = "./mozbuild"}
-"b3ddbcf" = {path = "./mozterm"}
-"38a4a9a" = {path = "./mozversioncontrol"}
-"26d92fb" = {path = "./../config/mozunit"}
-"cea2946" = {path = "./../testing/mozbase/manifestparser"}
-"ffcf6e6" = {path = "./../testing/mozbase/mozcrash"}
-"195ae2e" = {path = "./../testing/mozbase/mozdebug"}
-"8dab59a" = {path = "./../testing/mozbase/mozdevice"}
-"58d0848" = {path = "./../testing/mozbase/mozfile"}
-"fd0b608" = {path = "./../testing/mozbase/mozhttpd"}
-"7329809" = {path = "./../testing/mozbase/mozinfo"}
-"501835d" = {path = "./../testing/mozbase/mozinstall"}
-"807c1c5" = {path = "./../testing/mozbase/mozlog"}
-"e09e103" = {path = "./../testing/mozbase/moznetwork"}
-"132adec" = {path = "./../testing/mozbase/mozprocess"}
-"d88f467" = {path = "./../testing/mozbase/mozprofile"}
-"1de94f2" = {path = "./../testing/mozbase/mozrunner"}
-"6477f20" = {path = "./../testing/mozbase/moztest"}
-"f1d74ca" = {path = "./../testing/mozbase/mozversion"}
-"47200d8" = {path = "./../third_party/python/futures", markers="python_version < '3'"}
-"110bcc4" = {path = "./../third_party/python/jsmin"}
-"c49d32a" = {path = "./../third_party/python/mock-1.0.0", markers="python_version < '3.3'"}
-"c2c21d9" = {path = "./../third_party/python/py"}
-"f4b00e9" = {path = "./../third_party/python/pytest"}
-"053111f" = {path = "./../third_party/python/requests"}
-"d250320" = {path = "./../third_party/python/six"}
-"f1de77a" = {path = "./../third_party/python/which", markers="python_version < '3.3'"}
-
-[dev-packages]

python/Pipfile.lock

@@ -1,106 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "dfc219f64edc7715acdb35e03dcee665ec26908c18a58d3a3a88dda3ab393b17"
-        },
-        "pipfile-spec": 6,
-        "requires": {},
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "053111f": {
-            "path": "./../third_party/python/requests"
-        },
-        "110bcc4": {
-            "path": "./../third_party/python/jsmin"
-        },
-        "132adec": {
-            "path": "./../testing/mozbase/mozprocess"
-        },
-        "195ae2e": {
-            "path": "./../testing/mozbase/mozdebug"
-        },
-        "1de94f2": {
-            "path": "./../testing/mozbase/mozrunner"
-        },
-        "26d92fb": {
-            "path": "./../config/mozunit"
-        },
-        "38a4a9a": {
-            "path": "./mozversioncontrol"
-        },
-        "47200d8": {
-            "markers": "python_version < '3'",
-            "path": "./../third_party/python/futures"
-        },
-        "501835d": {
-            "path": "./../testing/mozbase/mozinstall"
-        },
-        "58d0848": {
-            "path": "./../testing/mozbase/mozfile"
-        },
-        "6477f20": {
-            "path": "./../testing/mozbase/moztest"
-        },
-        "7329809": {
-            "path": "./../testing/mozbase/mozinfo"
-        },
-        "807c1c5": {
-            "path": "./../testing/mozbase/mozlog"
-        },
-        "8dab59a": {
-            "path": "./../testing/mozbase/mozdevice"
-        },
-        "8ddb376": {
-            "path": "./mozbuild"
-        },
-        "b3ddbcf": {
-            "path": "./mozterm"
-        },
-        "c2c21d9": {
-            "path": "./../third_party/python/py"
-        },
-        "c49d32a": {
-            "markers": "python_version < '3.3'",
-            "path": "./../third_party/python/mock-1.0.0"
-        },
-        "cea2946": {
-            "path": "./../testing/mozbase/manifestparser"
-        },
-        "d250320": {
-            "path": "./../third_party/python/six"
-        },
-        "d5b4a14": {
-            "path": "./mach"
-        },
-        "d88f467": {
-            "path": "./../testing/mozbase/mozprofile"
-        },
-        "e09e103": {
-            "path": "./../testing/mozbase/moznetwork"
-        },
-        "f1d74ca": {
-            "path": "./../testing/mozbase/mozversion"
-        },
-        "f1de77a": {
-            "markers": "python_version < '3.3'",
-            "path": "./../third_party/python/which"
-        },
-        "f4b00e9": {
-            "path": "./../third_party/python/pytest"
-        },
-        "fd0b608": {
-            "path": "./../testing/mozbase/mozhttpd"
-        },
-        "ffcf6e6": {
-            "path": "./../testing/mozbase/mozcrash"
-        }
-    },
-    "develop": {}
-}

python/mach_commands.py

@@ -99,11 +99,8 @@ class MachCommands(MachCommandBase):
                          jobs=1,
                          three=False,
                          **kwargs):
-        if three:
+        pipenv_args = ['--three' if three else '--two']
-            # use pipenv to run tests against Python 3
+        self.activate_pipenv(pipfile=None, args=pipenv_args, populate=True)
-            self.activate_pipenv(os.path.join(here, 'Pipfile'), ['--three'])
-        else:
-            self._activate_virtualenv()
 
         if test_objects is None:
             from moztest.resolve import TestResolver

python/mozbuild/mozbuild/base.py

@@ -757,11 +757,11 @@ class MozbuildObject(ProcessExecutionMixin):
                 pipenv_reqs, require_hashes=False, vendored=True)
         return pipenv
 
-    def activate_pipenv(self, path, args=None):
+    def activate_pipenv(self, pipfile=None, args=None, populate=False):
-        if not os.path.exists(path):
+        if pipfile is not None and not os.path.exists(pipfile):
-            raise Exception('Pipfile not found: %s.' % path)
+            raise Exception('Pipfile not found: %s.' % pipfile)
         self.ensure_pipenv()
-        self.virtualenv_manager.activate_pipenv(path, args)
+        self.virtualenv_manager.activate_pipenv(pipfile, args, populate)
 
 
 class MachCommandBase(MozbuildObject):

python/mozbuild/mozbuild/virtualenv.py

@@ -20,9 +20,11 @@ IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\')
 IS_MSYS2 = (sys.platform == 'win32' and os.sep == '/')
 IS_CYGWIN = (sys.platform == 'cygwin')
 
-# Minimum version of Python required to build.
+# Minimum versions of Python required to build.
-MINIMUM_PYTHON_VERSION = LooseVersion('2.7.3')
+MINIMUM_PYTHON_VERSIONS = {
-MINIMUM_PYTHON_MAJOR = 2
+    2: LooseVersion('2.7.3'),
+    3: LooseVersion('3.5.0')
+}
 
 
 UPGRADE_WINDOWS = '''
@@ -38,6 +40,8 @@ another Python version. Ensure a modern Python can be found in the paths
 defined by the $PATH environment variable and try again.
 '''.lstrip()
 
+here = os.path.abspath(os.path.dirname(__file__))
+
 
 class VirtualenvManager(object):
     """Contains logic for managing virtualenvs for building the tree."""
@@ -207,7 +211,7 @@ class VirtualenvManager(object):
         return self.virtualenv_root
 
     def packages(self):
-        with file(self.manifest_path, 'rU') as fh:
+        with open(self.manifest_path, 'rU') as fh:
             packages = [line.rstrip().split(':')
                         for line in fh]
         return packages
@@ -250,7 +254,6 @@ class VirtualenvManager(object):
         environment is not configured properly, packages could be installed
         into the wrong place. This is how virtualenv's work.
         """
-
         packages = self.packages()
         python_lib = distutils.sysconfig.get_python_lib()
 
@@ -530,22 +533,34 @@ class VirtualenvManager(object):
         # force the virtualenv's interpreter to be used and all is well.
         # It /might/ be possible to cheat and set sys.executable to
         # self.python_path. However, this seems more risk than it's worth.
-        subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args,
+        pip = os.path.join(self.bin_path, 'pip')
-            stderr=subprocess.STDOUT)
+        subprocess.check_call([pip] + args, stderr=subprocess.STDOUT, cwd=self.topsrcdir)
 
-    def activate_pipenv(self, pipfile, args=None):
+    def activate_pipenv(self, pipfile=None, args=None, populate=False):
-        """Install a Pipfile located at path and activate environment"""
+        """Activate a virtual environment managed by pipenv
+
+        If ``pipfile`` is not ``None`` then the Pipfile located at the path
+        provided will be used to create the virtual environment. If
+        ``populate`` is ``True`` then the virtual environment will be
+        populated from the manifest file. The optional ``args`` list will be
+        passed to the pipenv commands.
+        """
         pipenv = os.path.join(self.bin_path, 'pipenv')
         env = os.environ.copy()
         env.update({
             'PIPENV_IGNORE_VIRTUALENVS': '1',
-            'PIPENV_PIPFILE': pipfile,
             'WORKON_HOME': os.path.join(self.topobjdir, '_virtualenvs'),
         })
 
         args = args or []
+
+        if pipfile is not None:
+            # Install from Pipfile
+            env['PIPENV_PIPFILE'] = pipfile
+            args.append('install')
+
         subprocess.check_call(
-            [pipenv, 'install'] + args,
+            [pipenv] + args,
             stderr=subprocess.STDOUT,
             env=env)
 
@@ -554,6 +569,13 @@ class VirtualenvManager(object):
             stderr=subprocess.STDOUT,
             env=env).rstrip()
 
+        if populate:
+            # Populate from the manifest
+            subprocess.check_call([
+                pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate',
+                self.topsrcdir, self.topobjdir, self.virtualenv_root, self.manifest_path],
+                stderr=subprocess.STDOUT, env=env)
+
         self.activate()
 
 
@@ -563,9 +585,10 @@ def verify_python_version(log_handle):
 
     our = LooseVersion('%d.%d.%d' % (major, minor, micro))
 
-    if major != MINIMUM_PYTHON_MAJOR or our < MINIMUM_PYTHON_VERSION:
+    if major not in MINIMUM_PYTHON_VERSIONS or our < MINIMUM_PYTHON_VERSIONS[major]:
-        log_handle.write('Python %s or greater (but not Python 3) is '
+        log_handle.write('The following Python versions are required to build:\n')
-            'required to build. ' % MINIMUM_PYTHON_VERSION)
+        for minver in MINIMUM_PYTHON_VERSIONS.values():
+            log_handle.write('* Python %s or greater\n' % minver)
         log_handle.write('You are running Python %s.\n' % our)
 
         if os.name in ('nt', 'ce'):