Bug 1674923 - Pass down virtualenv_manager from mach lint down to linters r=ahal

Before, on Windows, this resulted in installing the package in the parent environment (not the `virtualenv`). We fix this by passing down the `virtualenv_manager` so linters can install packages they need using that object's helper methods.

Differential Revision: https://phabricator.services.mozilla.com/D95738

python/mozlint/mozlint/cli.py

@@ -266,6 +266,7 @@ def run(
     setup=False,
     list_linters=False,
     num_procs=None,
+    virtualenv_manager=None,
     **lintargs
 ):
     from mozlint import LintRoller, formatters
@@ -285,7 +286,7 @@ def run(
     lint.read(find_linters(lintargs["config_paths"], linters))
 
     # Always run bootstrapping, but return early if --setup was passed in.
-    ret = lint.setup()
+    ret = lint.setup(virtualenv_manager=virtualenv_manager)
     if setup:
         return ret
 

python/mozlint/mozlint/roller.py

@@ -186,7 +186,7 @@ class LintRoller(object):
             linter.setdefault("exclude", []).extend(self.exclude)
             self.linters.append(linter)
 
-    def setup(self):
+    def setup(self, virtualenv_manager=None):
         """Run setup for applicable linters"""
         if not self.linters:
             raise LintersNotConfigured
@@ -198,6 +198,8 @@ class LintRoller(object):
             try:
                 setupargs = copy.deepcopy(self.lintargs)
                 setupargs["name"] = linter["name"]
+                if virtualenv_manager is not None:
+                    setupargs["virtualenv_manager"] = virtualenv_manager
                 start_time = time.time()
                 res = findobject(linter["setup"])(**setupargs)
                 self.log.debug(

python/mozlint/mozlint/util/pip.py

@@ -1,30 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import subprocess
-
-
-def _run_pip(*args):
-    """
-    Helper function that runs pip with subprocess
-    """
-    try:
-        subprocess.check_output(
-            ["pip"] + list(args), stderr=subprocess.STDOUT, universal_newlines=True
-        )
-        return True
-    except subprocess.CalledProcessError as e:
-        print(e.output)
-        return False
-
-
-def reinstall_program(REQ_PATH):
-    """
-    Try to install flake8 at the target version, returns True on success
-    otherwise prints the otuput of the pip command and returns False
-    """
-    if _run_pip("install", "-U", "--require-hashes", "-r", REQ_PATH):
-        return True
-
-    return False

tools/lint/mach_commands.py

@@ -88,6 +88,7 @@ class MachCommands(MachCommandBase):
         lintargs["exclude"] = get_global_excludes(lintargs["root"])
         lintargs["config_paths"].insert(0, here)
         lintargs["virtualenv_bin_path"] = self.virtualenv_manager.bin_path
+        lintargs["virtualenv_manager"] = self.virtualenv_manager
         for path in EXCLUSION_FILES:
             parser.GLOBAL_SUPPORT_FILES.append(os.path.join(self.topsrcdir, path))
         return cli.run(*runargs, **lintargs)

tools/lint/python/black.py

@@ -13,7 +13,6 @@ import sys
 
 from mozlint import result
 from mozlint.pathutils import expand_exclusions
-from mozlint.util import pip
 from mozprocess import ProcessHandler
 
 here = os.path.abspath(os.path.dirname(__file__))
@@ -109,7 +108,10 @@ def run_process(config, cmd):
 
 
 def setup(root, **lintargs):
-    if not pip.reinstall_program(BLACK_REQUIREMENTS_PATH):
+    virtualenv_manager = lintargs["virtualenv_manager"]
+    try:
+        virtualenv_manager.install_pip_requirements(BLACK_REQUIREMENTS_PATH, quiet=True)
+    except subprocess.CalledProcessError:
         print(BLACK_INSTALL_ERROR)
         return 1
 

tools/lint/python/flake8.py

@@ -13,7 +13,6 @@ import mozpack.path as mozpath
 
 from mozlint import result
 from mozlint.pathutils import expand_exclusions
-from mozlint.util import pip
 
 here = os.path.abspath(os.path.dirname(__file__))
 FLAKE8_REQUIREMENTS_PATH = os.path.join(here, "flake8_requirements.txt")
@@ -76,7 +75,12 @@ class NothingToLint(Exception):
 
 
 def setup(root, **lintargs):
-    if not pip.reinstall_program(FLAKE8_REQUIREMENTS_PATH):
+    virtualenv_manager = lintargs["virtualenv_manager"]
+    try:
+        virtualenv_manager.install_pip_requirements(
+            FLAKE8_REQUIREMENTS_PATH, quiet=True
+        )
+    except subprocess.CalledProcessError:
         print(FLAKE8_INSTALL_ERROR)
         return 1
 

tools/lint/python/pylint.py

@@ -4,6 +4,7 @@
 
 import json
 import os
+import subprocess
 
 import signal
 
@@ -11,7 +12,6 @@ from mozprocess import ProcessHandler
 
 from mozlint import result
 from mozlint.pathutils import expand_exclusions
-from mozlint.util import pip
 
 here = os.path.abspath(os.path.dirname(__file__))
 PYLINT_REQUIREMENTS_PATH = os.path.join(here, "pylint_requirements.txt")
@@ -48,7 +48,12 @@ class PylintProcess(ProcessHandler):
 
 
 def setup(root, **lintargs):
-    if not pip.reinstall_program(PYLINT_REQUIREMENTS_PATH):
+    virtualenv_manager = lintargs["virtualenv_manager"]
+    try:
+        virtualenv_manager.install_pip_requirements(
+            PYLINT_REQUIREMENTS_PATH, quiet=True
+        )
+    except subprocess.CalledProcessError:
         print(PYLINT_INSTALL_ERROR)
         return 1
 

tools/lint/rst/__init__.py

@@ -8,7 +8,6 @@ import subprocess
 
 from mozlint import result
 from mozlint.pathutils import expand_exclusions
-from mozlint.util import pip
 from mozfile import which
 
 # Error Levels
@@ -43,7 +42,12 @@ RSTCHECK_FORMAT_REGEX = re.compile(r"(.*):(.*): \(.*/([0-9]*)\) (.*)$")
 
 
 def setup(root, **lintargs):
-    if not pip.reinstall_program(rstcheck_requirements_file):
+    virtualenv_manager = lintargs["virtualenv_manager"]
+    try:
+        virtualenv_manager.install_pip_requirements(
+            rstcheck_requirements_file, quiet=True
+        )
+    except subprocess.CalledProcessError:
         print(RSTCHECK_INSTALL_ERROR)
         return 1
 

tools/lint/spell/__init__.py

@@ -4,6 +4,7 @@
 
 import os
 import re
+import subprocess
 
 # py2-compat
 try:
@@ -14,7 +15,6 @@ except ImportError:
 from mozfile import which
 
 from mozlint import result
-from mozlint.util import pip
 from mozlint.util.implementation import LintProcess
 
 here = os.path.abspath(os.path.dirname(__file__))
@@ -87,7 +87,12 @@ def get_codespell_binary():
 
 
 def setup(root, **lintargs):
-    if not pip.reinstall_program(CODESPELL_REQUIREMENTS_PATH):
+    virtualenv_manager = lintargs["virtualenv_manager"]
+    try:
+        virtualenv_manager.install_pip_requirements(
+            CODESPELL_REQUIREMENTS_PATH, quiet=True
+        )
+    except subprocess.CalledProcessError:
         print(CODESPELL_INSTALL_ERROR)
         return 1
 

tools/lint/test/conftest.py

@@ -15,7 +15,7 @@ from mozpack import path
 import pytest
 
 here = path.abspath(path.dirname(__file__))
-build = MozbuildObject.from_environment(cwd=here)
+build = MozbuildObject.from_environment(cwd=here, virtualenv_name="python-test")
 
 lintdir = path.dirname(here)
 sys.path.insert(0, lintdir)
@@ -94,7 +94,7 @@ def run_setup(config):
         return
 
     func = findobject(config["setup"])
-    func(build.topsrcdir)
+    func(build.topsrcdir, virtualenv_manager=build.virtualenv_manager)
 
 
 @pytest.fixture