diff --git a/.flake8 b/.flake8 deleted file mode 100644 index a27dfabab003..000000000000 --- a/.flake8 +++ /dev/null @@ -1,130 +0,0 @@ -[flake8] -max-line-length = 99 -exclude = - # These paths should be triaged and either fixed or moved to the list below. - devtools/shared, - dom/bindings/Codegen.py, - dom/bindings/parser/WebIDL.py, - dom/bindings/parser/tests/test_arraybuffer.py, - dom/bindings/parser/tests/test_securecontext_extended_attribute.py, - gfx/tests, - ipc/ipdl/ipdl, - layout/base/tests/marionette, - layout/reftests/border-image, - layout/reftests/fonts, - layout/reftests/w3c-css, - layout/style, - media/libdav1d/generate_source.py, - moz.configure, - netwerk/dns/prepare_tlds.py, - netwerk/protocol/http/make_incoming_tables.py, - python/l10n/fluent_migrations, - security/manager/ssl/tests/unit, - servo/components/style, - testing/condprofile/condprof/android.py, - testing/condprofile/condprof/creator.py, - testing/condprofile/condprof/desktop.py, - testing/condprofile/condprof/runner.py, - testing/condprofile/condprof/scenarii/heavy.py, - testing/condprofile/condprof/scenarii/settled.py, - testing/condprofile/condprof/scenarii/synced.py - testing/condprofile/condprof/helpers.py, - testing/jsshell/benchmark.py, - testing/marionette/mach_commands.py, - testing/mozharness/docs, - testing/mozharness/examples, - testing/mozharness/external_tools, - testing/mozharness/mach_commands.py, - testing/mozharness/manifestparser, - testing/mozharness/mozprocess, - testing/mozharness/setup.py, - testing/parse_build_tests_ccov.py, - testing/runtimes/writeruntimes.py, - testing/tools/iceserver/iceserver.py, - testing/tools/websocketprocessbridge/websocketprocessbridge.py, - toolkit/components/featuregates, - toolkit/content/tests/chrome/file_about_networking_wsh.py, - toolkit/library/build/dependentlibs.py, - toolkit/locales/generate_update_locale.py, - toolkit/mozapps, - toolkit/moz.configure, - toolkit/nss.configure, - - # mako files are not really python files - *.mako.py, - - # These paths are intentionally excluded (not necessarily for good reason). - build/moz.configure/*.configure, - build/pymake/, - browser/extensions/mortar/ppapi/, - browser/moz.configure, - dom/canvas/test/webgl-conf/checkout/closure-library/, - editor/libeditor/tests/browserscope/, - intl/icu/, - ipc/chromium/src/third_party/, - js/*.configure, - gfx/angle/, - gfx/harfbuzz, - gfx/skia/, - memory/moz.configure, - mobile/android/*.configure, - node_modules, - python/mozbuild/mozbuild/test/configure/data, - security/nss/, - testing/marionette/harness/marionette_harness/runner/mixins, - testing/marionette/harness/marionette_harness/tests, - testing/mochitest/pywebsocket3, - testing/mozharness/configs/test/test_malformed.py, - testing/web-platform/tests, - tools/lint/test/files, - tools/crashreporter/*.configure, - .ycm_extra_conf.py, - -# See: -# - http://flake8.pycqa.org/en/latest/user/error-codes.html -# - http://pep8.readthedocs.io/en/latest/intro.html#configuration -ignore = - # These should be triaged and either fixed or moved to the list below. - W605, W606, - # These are intentionally disabled (not necessarily for good reason). - # F723: syntax error in type comment - # text contains quotes which breaks our custom JSON formatter - F723, E704, E741, - - # black is already in charge of formatting, no need to start a formatter - # battle here - E1, W1, E2, W2, E3, W3, E4, W4, E5, W5 - -per-file-ignores = - # These paths are intentionally excluded. - ipc/ipdl/*: F403, F405 - layout/tools/reftest/selftest/conftest.py: F811 - # cpp_eclipse has a lot of multi-line embedded XML which exceeds line length - python/mozbuild/mozbuild/backend/cpp_eclipse.py: E501 - testing/firefox-ui/**/__init__.py: F401 - testing/marionette/**/__init__.py: F401 - testing/mochitest/tests/python/conftest.py: F811 - testing/mozbase/manifestparser/tests/test_filters.py: E731 - testing/mozbase/mozlog/tests/test_formatters.py: E501 - testing/mozharness/configs/*: E124, E127, E128, E131, E231, E261, E265, E266, E501, W391 - - # These paths contain Python-2 only syntax which cause errors since flake8 - # is run with Python 3. - build/compare-mozconfig/compare-mozconfigs.py: F821 - build/midl.py: F821 - build/pgo/genpgocert.py: F821 - config/MozZipFile.py: F821 - config/check_source_count.py: F821 - config/tests/unitMozZipFile.py: F821 - ipc/pull-chromium.py: F633 - js/src/**: F633, F821 - python/mozbuild/mozbuild/action/dump_env.py: F821 - python/mozbuild/mozbuild/dotproperties.py: F821 - python/mozbuild/mozbuild/testing.py: F821 - python/mozbuild/mozbuild/util.py: F821 - testing/mozharness/mozharness/mozilla/testing/android.py: F821 - testing/mochitest/runtests.py: F821 - -builtins = - # For GDB extensions - gdb diff --git a/.hgignore b/.hgignore index f3b1e406eb54..21559a945dfd 100644 --- a/.hgignore +++ b/.hgignore @@ -225,6 +225,9 @@ _OPT\.OBJ/ # Unit test \.pytest_cache/ +# Ruff +\.ruff_cache/ + # Ignore files created when running a reftest. ^lextab.py$ diff --git a/docs/code-quality/index.rst b/docs/code-quality/index.rst index 305d5719a021..d45ce7e06be3 100644 --- a/docs/code-quality/index.rst +++ b/docs/code-quality/index.rst @@ -90,11 +90,11 @@ In this document, we try to list these all tools. - Meta bug - More info - Upstream - * - Flake8 - - Yes (with `autopep8 `_) - - `bug 1155970 `__ - - :ref:`Flake8` - - http://flake8.pycqa.org/ + * - ruff + - Yes + - `bug 1811850 `__ + - :ref:`ruff` + - https://github.com/charliermarsh/ruff * - black - Yes - `bug 1555560 `__ @@ -105,12 +105,6 @@ In this document, we try to list these all tools. - `bug 1623024 `__ - :ref:`pylint` - https://www.pylint.org/ - * - Python 2/3 compatibility check - - - - `bug 1496527 `__ - - :ref:`Python 2/3 compatibility check` - - - .. list-table:: Rust :widths: 20 20 20 20 20 diff --git a/docs/code-quality/lint/linters/flake8.rst b/docs/code-quality/lint/linters/flake8.rst deleted file mode 100644 index 0e46d33ab026..000000000000 --- a/docs/code-quality/lint/linters/flake8.rst +++ /dev/null @@ -1,46 +0,0 @@ -Flake8 -====== - -`Flake8 `__ is a popular lint wrapper for python. Under the hood, it runs three other tools and -combines their results: - -* `pep8 `__ for checking style -* `pyflakes `__ for checking syntax -* `mccabe `__ for checking complexity - - -Run Locally ------------ - -The mozlint integration of flake8 can be run using mach: - -.. parsed-literal:: - - $ mach lint --linter flake8 - -Alternatively, omit the ``--linter flake8`` and run all configured linters, which will include -flake8. - - -Configuration -------------- - -Path configuration is defined in the root `.flake8`_ file. Please update this file rather than -``tools/lint/flake8.yml`` if you need to exclude a new path. For an overview of the supported -configuration, see `flake8's documentation`_. - -.. _.flake8: https://searchfox.org/mozilla-central/source/.flake8 -.. _flake8's documentation: https://flake8.pycqa.org/en/latest/user/configuration.html - -Autofix -------- - -The flake8 linter provides a ``--fix`` option. It is based on `autopep8 `__. -Please note that autopep8 does NOT fix all issues reported by flake8. - - -Sources -------- - -* `Configuration (YAML) `_ -* `Source `_ diff --git a/docs/code-quality/lint/linters/ruff.rst b/docs/code-quality/lint/linters/ruff.rst new file mode 100644 index 000000000000..359e8dcbf686 --- /dev/null +++ b/docs/code-quality/lint/linters/ruff.rst @@ -0,0 +1,44 @@ +Ruff +==== + +`Ruff `_ is an extremely fast Python +linter and formatter, written in Rust. It can process all of mozilla-central in +under a second, and implements rule sets from a large array of Python linters +and formatters, including: + +* flake8 (pycodestyle, pyflakes and mccabe) +* isort +* pylint +* pyupgrade +* and many many more! + +Run Locally +----------- + +The mozlint integration of ruff can be run using mach: + +.. parsed-literal:: + + $ mach lint --linter ruff + + +Configuration +------------- + +Ruff is configured in the root `pyproject.toml`_ file. Additionally, ruff will +pick up any ``pyproject.toml`` or ``ruff.toml`` files in subdirectories. The +settings in these files will only apply to files contained within these +subdirs. For more details on configuration discovery, see the `configuration +documentation`_. + +For a list of options, see the `settings documentation`_. + +Sources +------- + +* `Configuration (YAML) `_ +* `Source `_ + +.. _pyproject.toml: https://searchfox.org/mozilla-central/source/pyproject.toml +.. _configuration documentation: https://beta.ruff.rs/docs/configuration/ +.. _settings documentation: https://beta.ruff.rs/docs/settings/ diff --git a/moz.build b/moz.build index 80e23e9e7c07..e33ad8e4a082 100644 --- a/moz.build +++ b/moz.build @@ -35,6 +35,9 @@ with Files("docs/**"): with Files("mach*"): BUG_COMPONENT = ("Firefox Build System", "Mach Core") +with Files("pyproject.toml"): + BUG_COMPONENT = ("Developer Infrastructure", "Lint and Formatting") + with Files("*moz*"): BUG_COMPONENT = ("Firefox Build System", "General") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000000..bfc656b9668d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,127 @@ +[tool.ruff] +line-length = 99 +# See https://beta.ruff.rs/docs/rules/ for a full list of rules. +select = [ + "E", "W", # pycodestyle + "F", # pyflakes +] +ignore = [ + # These should be triaged and either fixed or moved to the list below. + "E713", "E714", "W605", + + # These are intentionally ignored (not necessarily for good reason). + "E741", + + # These are handled by black. + "E1", "E4", "E5", "W2", "W5" +] +builtins = ["gdb"] +exclude = [ + # These paths should be triaged and either fixed or moved to the list below. + "devtools/shared", + "dom/bindings/Codegen.py", + "dom/bindings/parser/WebIDL.py", + "dom/bindings/parser/tests/test_arraybuffer.py", + "dom/bindings/parser/tests/test_securecontext_extended_attribute.py", + "gfx/tests", + "ipc/ipdl/ipdl", + "layout/base/tests/marionette", + "layout/reftests/border-image", + "layout/reftests/fonts", + "layout/reftests/w3c-css", + "layout/style", + "media/libdav1d/generate_source.py", + "moz.configure", + "netwerk/dns/prepare_tlds.py", + "netwerk/protocol/http/make_incoming_tables.py", + "python/l10n/fluent_migrations", + "security/manager/ssl/tests/unit", + "servo/components/style", + "testing/condprofile/condprof/android.py", + "testing/condprofile/condprof/creator.py", + "testing/condprofile/condprof/desktop.py", + "testing/condprofile/condprof/runner.py", + "testing/condprofile/condprof/scenarii/heavy.py", + "testing/condprofile/condprof/scenarii/settled.py", + "testing/condprofile/condprof/scenarii/synced.p", + "testing/condprofile/condprof/helpers.py", + "testing/jsshell/benchmark.py", + "testing/marionette/mach_commands.py", + "testing/mozharness/docs", + "testing/mozharness/examples", + "testing/mozharness/external_tools", + "testing/mozharness/mach_commands.py", + "testing/mozharness/manifestparser", + "testing/mozharness/mozprocess", + "testing/mozharness/setup.py", + "testing/parse_build_tests_ccov.py", + "testing/runtimes/writeruntimes.py", + "testing/tools/iceserver/iceserver.py", + "testing/tools/websocketprocessbridge/websocketprocessbridge.py", + "toolkit/components/featuregates", + "toolkit/content/tests/chrome/file_about_networking_wsh.py", + "toolkit/library/build/dependentlibs.py", + "toolkit/locales/generate_update_locale.py", + "toolkit/mozapps", + "toolkit/moz.configure", + "toolkit/nss.configure", + + # mako files are not really python files + "*.mako.py", + + # These paths are intentionally excluded (not necessarily for good reason). + "build/moz.configure/*.configure", + "build/pymake/", + "browser/extensions/mortar/ppapi/", + "browser/moz.configure", + "dom/canvas/test/webgl-conf/checkout/closure-library/", + "editor/libeditor/tests/browserscope/", + "intl/icu/", + "ipc/chromium/src/third_party/", + "js/*.configure", + "gfx/angle/", + "gfx/harfbuzz", + "gfx/skia/", + "memory/moz.configure", + "mobile/android/*.configure", + "node_modules", + "python/mozbuild/mozbuild/test/configure/data", + "security/nss/", + "testing/marionette/harness/marionette_harness/runner/mixins", + "testing/marionette/harness/marionette_harness/tests", + "testing/mochitest/pywebsocket3", + "testing/mozharness/configs/test/test_malformed.py", + "testing/web-platform/tests", + "tools/lint/test/files", + "tools/crashreporter/*.configure", + ".ycm_extra_conf.py", +] + +[tool.ruff.per-file-ignores] +# These paths are intentionally excluded. +"ipc/ipdl/*" = ["F403", "F405"] +"layout/tools/reftest/selftest/conftest.py" = ["F811"] +# cpp_eclipse has a lot of multi-line embedded XML which exceeds line length +"python/mozbuild/mozbuild/backend/cpp_eclipse.py" = ["E501"] +"testing/firefox-ui/**/__init__.py" = ["F401"] +"testing/marionette/**/__init__.py" = ["F401"] +"testing/mochitest/tests/python/conftest.py" = ["F811"] +"testing/mozbase/manifestparser/tests/test_filters.py" = ["E731"] +"testing/mozbase/mozlog/tests/test_formatters.py" = ["E501"] +"testing/mozharness/configs/*" = ["E501"] +"**/*.configure" = ["F821"] +# These paths contain Python-2 only syntax. +"build/compare-mozconfig/compare-mozconfigs.py" = ["F821"] +"build/midl.py" = ["F821"] +"build/pgo/genpgocert.py" = ["F821"] +"config/MozZipFile.py" = ["F821"] +"config/check_source_count.py" = ["F821"] +"config/tests/unitMozZipFile.py" = ["F821"] +"ipc/pull-chromium.py" = ["F633"] +"js/src/**" = ["F633", "F821"] +"python/mozbuild/mozbuild/action/dump_env.py" = ["F821"] +"python/mozbuild/mozbuild/dotproperties.py" = ["F821"] +"python/mozbuild/mozbuild/testing.py" = ["F821"] +"python/mozbuild/mozbuild/util.py" = ["F821"] +"testing/mozharness/mozharness/mozilla/testing/android.py" = ["F821"] +"testing/mochitest/runtests.py" = ["F821"] diff --git a/python/gdbpp/gdbpp/__init__.py b/python/gdbpp/gdbpp/__init__.py index 7a7681aa6de3..376061b679e9 100644 --- a/python/gdbpp/gdbpp/__init__.py +++ b/python/gdbpp/gdbpp/__init__.py @@ -20,12 +20,12 @@ class GeckoPrettyPrinter(object): return wrapped -import gdbpp.enumset # NOQA: F401 -import gdbpp.linkedlist # NOQA: F401 -import gdbpp.owningthread # NOQA: F401 -import gdbpp.smartptr # NOQA: F401 -import gdbpp.string # NOQA: F401 -import gdbpp.tarray # NOQA: F401 -import gdbpp.thashtable # NOQA: F401 +import gdbpp.enumset # noqa: F401 +import gdbpp.linkedlist # noqa: F401 +import gdbpp.owningthread # noqa: F401 +import gdbpp.smartptr # noqa: F401 +import gdbpp.string # noqa: F401 +import gdbpp.tarray # noqa: F401 +import gdbpp.thashtable # noqa: F401 gdb.printing.register_pretty_printer(None, GeckoPrettyPrinter.pp) diff --git a/taskcluster/ci/source-test/mozlint.yml b/taskcluster/ci/source-test/mozlint.yml index c3ab520672f6..6075ebc2e915 100644 --- a/taskcluster/ci/source-test/mozlint.yml +++ b/taskcluster/ci/source-test/mozlint.yml @@ -242,20 +242,6 @@ mscom-init: - '**/*.h' - 'tools/lint/mscom-init.yml' -py-flake8: - description: flake8 run over the gecko codebase - treeherder: - symbol: py(f8) - run: - mach: lint -v -l flake8 -f treeherder -f json:/builds/worker/mozlint.json * - when: - files-changed: - - '**/*.py' - - '**/.flake8' - - 'tools/lint/flake8.yml' - # moz.configure files are also Python files. - - '**/*.configure' - py-black: description: black run over the gecko codebase treeherder: @@ -272,6 +258,22 @@ py-black: - 'pyproject.toml' - 'tools/lint/black.yml' +py-ruff: + description: Run ruff over the gecko codebase + treeherder: + symbol: py(ruff) + run: + mach: lint -v -l ruff -f treeherder -f json:/builds/worker/mozlint.json * + when: + files-changed: + - '**/*.py' + - '**/*.configure' + - '**/.ruff.toml' + - 'pyproject.toml' + - 'tools/lint/ruff.yml' + - 'tools/lint/python/ruff.py' + - 'tools/lint/python/ruff_requirements.txt' + py-pylint: description: pylint run over the gecko codebase treeherder: diff --git a/tools/lint/file-whitespace.yml b/tools/lint/file-whitespace.yml index e598a28414a0..2e9035da5899 100644 --- a/tools/lint/file-whitespace.yml +++ b/tools/lint/file-whitespace.yml @@ -3,9 +3,9 @@ file-whitespace: description: File content sanity check include: - . - - tools/lint/python/flake8_requirements.txt - tools/lint/python/pylint_requirements.txt - tools/lint/python/black_requirements.txt + - tools/lint/python/ruff_requirements.txt - tools/lint/rst/requirements.txt - tools/lint/tox/tox_requirements.txt - tools/lint/spell/codespell_requirements.txt diff --git a/tools/lint/flake8.yml b/tools/lint/flake8.yml deleted file mode 100644 index 11d31932717c..000000000000 --- a/tools/lint/flake8.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -flake8: - description: Python linter - # Excludes should be added to topsrcdir/.flake8. - exclude: [] - # The configure option is used by the build system - extensions: ['configure', 'py'] - support-files: - - '**/.flake8' - - 'tools/lint/python/flake8*' - # Rules that should result in warnings rather than errors. - warning-rules: [] - type: external - payload: python.flake8:lint - setup: python.flake8:setup diff --git a/tools/lint/python/flake8.py b/tools/lint/python/flake8.py deleted file mode 100644 index 88fec87822dd..000000000000 --- a/tools/lint/python/flake8.py +++ /dev/null @@ -1,215 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import json -import os -import platform -import subprocess -import sys - -import mozfile -import mozpack.path as mozpath -from mozlint import result -from mozlint.pathutils import expand_exclusions - -here = os.path.abspath(os.path.dirname(__file__)) -FLAKE8_REQUIREMENTS_PATH = os.path.join(here, "flake8_requirements.txt") - -FLAKE8_NOT_FOUND = """ -Could not find flake8! Install flake8 and try again. - - $ pip install -U --require-hashes -r {} -""".strip().format( - FLAKE8_REQUIREMENTS_PATH -) - - -FLAKE8_INSTALL_ERROR = """ -Unable to install correct version of flake8 -Try to install it manually with: - $ pip install -U --require-hashes -r {} -""".strip().format( - FLAKE8_REQUIREMENTS_PATH -) - -LINE_OFFSETS = { - # continuation line under-indented for hanging indent - "E121": (-1, 2), - # continuation line missing indentation or outdented - "E122": (-1, 2), - # continuation line over-indented for hanging indent - "E126": (-1, 2), - # continuation line over-indented for visual indent - "E127": (-1, 2), - # continuation line under-indented for visual indent - "E128": (-1, 2), - # continuation line unaligned for hanging indend - "E131": (-1, 2), - # expected 1 blank line, found 0 - "E301": (-1, 2), - # expected 2 blank lines, found 1 - "E302": (-2, 3), -} -"""Maps a flake8 error to a lineoffset tuple. - -The offset is of the form (lineno_offset, num_lines) and is passed -to the lineoffset property of an `Issue`. -""" - - -def default_bindir(): - # We use sys.prefix to find executables as that gets modified with - # virtualenv's activate_this.py, whereas sys.executable doesn't. - if platform.system() == "Windows": - return os.path.join(sys.prefix, "Scripts") - else: - return os.path.join(sys.prefix, "bin") - - -class NothingToLint(Exception): - """Exception used to bail out of flake8's internals if all the specified - files were excluded. - """ - - -def setup(root, **lintargs): - virtualenv_manager = lintargs["virtualenv_manager"] - try: - virtualenv_manager.install_pip_requirements( - FLAKE8_REQUIREMENTS_PATH, quiet=True - ) - except subprocess.CalledProcessError: - print(FLAKE8_INSTALL_ERROR) - return 1 - - -def lint(paths, config, **lintargs): - - root = lintargs["root"] - virtualenv_bin_path = lintargs.get("virtualenv_bin_path") - config_path = os.path.join(root, ".flake8") - - results = run(paths, config, **lintargs) - fixed = 0 - - if lintargs.get("fix"): - # fix and run again to count remaining issues - fixed = len(results) - fix_cmd = [ - os.path.join(virtualenv_bin_path or default_bindir(), "autopep8"), - "--global-config", - config_path, - "--in-place", - "--recursive", - ] - - if config.get("exclude"): - fix_cmd.extend(["--exclude", ",".join(config["exclude"])]) - - subprocess.call(fix_cmd + paths) - - results = run(paths, config, **lintargs) - - fixed = fixed - len(results) - - return {"results": results, "fixed": fixed} - - -def run(paths, config, **lintargs): - from flake8 import __version__ as flake8_version - from flake8.main.application import Application - - log = lintargs["log"] - root = lintargs["root"] - config_path = os.path.join(root, ".flake8") - - # Run flake8. - app = Application() - log.debug("flake8 version={}".format(flake8_version)) - - output_file = mozfile.NamedTemporaryFile(mode="r") - flake8_cmd = [ - "--config", - config_path, - "--output-file", - output_file.name, - "--format", - '{"path":"%(path)s","lineno":%(row)s,' - '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}', - "--filename", - ",".join(["*.{}".format(e) for e in config["extensions"]]), - ] - log.debug("Command: {}".format(" ".join(flake8_cmd))) - - orig_make_file_checker_manager = app.make_file_checker_manager - - def wrap_make_file_checker_manager(self): - """Flake8 is very inefficient when it comes to applying exclusion - rules, using `expand_exclusions` to turn directories into a list of - relevant python files is an order of magnitude faster. - - Hooking into flake8 here also gives us a convenient place to merge the - `exclude` rules specified in the root .flake8 with the ones added by - tools/lint/mach_commands.py. - """ - # Ignore exclude rules if `--no-filter` was passed in. - config.setdefault("exclude", []) - if lintargs.get("use_filters", True): - config["exclude"].extend(map(mozpath.normpath, self.options.exclude)) - - # Since we use the root .flake8 file to store exclusions, we haven't - # properly filtered the paths through mozlint's `filterpaths` function - # yet. This mimics that though there could be other edge cases that are - # different. Maybe we should call `filterpaths` directly, though for - # now that doesn't appear to be necessary. - filtered = [ - p for p in paths if not any(p.startswith(e) for e in config["exclude"]) - ] - - self.options.filenames = self.options.filenames + list( - expand_exclusions(filtered, config, root) - ) - - if not self.options.filenames: - raise NothingToLint - return orig_make_file_checker_manager() - - app.make_file_checker_manager = wrap_make_file_checker_manager.__get__( - app, Application - ) - - # Make sure to run from repository root so exclusions are joined to the - # repository root and not the current working directory. - oldcwd = os.getcwd() - os.chdir(root) - try: - app.run(flake8_cmd) - except NothingToLint: - pass - finally: - os.chdir(oldcwd) - - results = [] - - WARNING_RULES = set(config.get("warning-rules", [])) - - def process_line(line): - # Escape slashes otherwise JSON conversion will not work - line = line.replace("\\", "\\\\") - try: - res = json.loads(line) - except ValueError: - print("Non JSON output from linter, will not be processed: {}".format(line)) - return - - if res.get("code") in LINE_OFFSETS: - res["lineoffset"] = LINE_OFFSETS[res["code"]] - - if res["rule"] in WARNING_RULES: - res["level"] = "warning" - - results.append(result.from_config(config, **res)) - - list(map(process_line, output_file.readlines())) - return results diff --git a/tools/lint/python/flake8_requirements.in b/tools/lint/python/flake8_requirements.in deleted file mode 100644 index 0a9262b9c6e2..000000000000 --- a/tools/lint/python/flake8_requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -flake8==5.0.4 -zipp==0.5 -autopep8==1.7.0 -typing-extensions==3.10.0.2 diff --git a/tools/lint/python/flake8_requirements.txt b/tools/lint/python/flake8_requirements.txt deleted file mode 100644 index 36879d20c839..000000000000 --- a/tools/lint/python/flake8_requirements.txt +++ /dev/null @@ -1,41 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: -# -# pip-compile --generate-hashes tools/lint/python/flake8_requirements.in -# -autopep8==1.7.0 \ - --hash=sha256:6f09e90a2be784317e84dc1add17ebfc7abe3924239957a37e5040e27d812087 \ - --hash=sha256:ca9b1a83e53a7fad65d731dc7a2a2d50aa48f43850407c59f6a1a306c4201142 - # via -r tools/lint/python/flake8_requirements.in -flake8==5.0.4 \ - --hash=sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db \ - --hash=sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248 - # via -r tools/lint/python/flake8_requirements.in -mccabe==0.7.0 \ - --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \ - --hash=sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e - # via flake8 -pycodestyle==2.9.1 \ - --hash=sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785 \ - --hash=sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b - # via - # autopep8 - # flake8 -pyflakes==2.5.0 \ - --hash=sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2 \ - --hash=sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3 - # via flake8 -toml==0.10.2 \ - --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ - --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f - # via autopep8 -typing-extensions==3.10.0.2 \ - --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ - --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ - --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 - # via -r tools/lint/python/flake8_requirements.in -zipp==0.5 \ - --hash=sha256:46dfd547d9ccbf8bdc26ecea52818046bb28509f12bb6a0de1cd66ab06e9a9be \ - --hash=sha256:d7ac25f895fb65bff937b381353c14eb1fa23d35f40abd72a5342cd57eb57fd1 - # via -r tools/lint/python/flake8_requirements.in diff --git a/tools/lint/python/ruff.py b/tools/lint/python/ruff.py new file mode 100644 index 000000000000..93916463fa00 --- /dev/null +++ b/tools/lint/python/ruff.py @@ -0,0 +1,177 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import platform +import re +import signal +import subprocess +import sys +from pathlib import Path + +import mozfile +from mozlint import result +from mozprocess.processhandler import ProcessHandler + +here = os.path.abspath(os.path.dirname(__file__)) +RUFF_REQUIREMENTS_PATH = os.path.join(here, "ruff_requirements.txt") + +RUFF_NOT_FOUND = """ +Could not find ruff! Install ruff and try again. + + $ pip install -U --require-hashes -r {} +""".strip().format( + RUFF_REQUIREMENTS_PATH +) + + +RUFF_INSTALL_ERROR = """ +Unable to install correct version of ruff! +Try to install it manually with: + $ pip install -U --require-hashes -r {} +""".strip().format( + RUFF_REQUIREMENTS_PATH +) + + +def default_bindir(): + # We use sys.prefix to find executables as that gets modified with + # virtualenv's activate_this.py, whereas sys.executable doesn't. + if platform.system() == "Windows": + return os.path.join(sys.prefix, "Scripts") + else: + return os.path.join(sys.prefix, "bin") + + +def get_ruff_version(binary): + """ + Returns found binary's version + """ + try: + output = subprocess.check_output( + [binary, "--version"], + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError as e: + output = e.output + + matches = re.match(r"ruff ([0-9\.]+)", output) + if matches: + return matches[1] + print("Error: Could not parse the version '{}'".format(output)) + + +def setup(root, log, **lintargs): + virtualenv_bin_path = lintargs.get("virtualenv_bin_path") + binary = mozfile.which("ruff", path=(virtualenv_bin_path, default_bindir())) + + if binary and os.path.isfile(binary): + log.debug(f"Looking for ruff at {binary}") + version = get_ruff_version(binary) + versions = [ + line.split()[0].strip() + for line in open(RUFF_REQUIREMENTS_PATH).readlines() + if line.startswith("ruff==") + ] + if [f"ruff=={version}"] == versions: + log.debug("ruff is present with expected version {}".format(version)) + return 0 + else: + log.debug("ruff is present but unexpected version {}".format(version)) + + virtualenv_manager = lintargs["virtualenv_manager"] + try: + virtualenv_manager.install_pip_requirements(RUFF_REQUIREMENTS_PATH, quiet=True) + except subprocess.CalledProcessError: + print(RUFF_INSTALL_ERROR) + return 1 + + +class RuffProcess(ProcessHandler): + def __init__(self, config, *args, **kwargs): + self.config = config + self.stderr = [] + kwargs["stream"] = False + kwargs["universal_newlines"] = True + kwargs["processStderrLine"] = lambda line: print(line, file=sys.stderr) + ProcessHandler.__init__(self, *args, **kwargs) + + def run(self, *args, **kwargs): + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + ProcessHandler.run(self, *args, **kwargs) + signal.signal(signal.SIGINT, orig) + + +def run_process(config, cmd): + proc = RuffProcess(config, cmd) + proc.run() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + + return "\n".join(proc.output) + + +def lint(paths, config, log, **lintargs): + fixed = 0 + results = [] + + if not paths: + return {"results": results, "fixed": fixed} + + # Currently ruff only lints non `.py` files if they are explicitly passed + # in. So we need to find any non-py files manually. This can be removed + # after https://github.com/charliermarsh/ruff/issues/3410 is fixed. + exts = [e for e in config["extensions"] if e != "py"] + non_py_files = [] + for path in paths: + p = Path(path) + if not p.is_dir(): + continue + for ext in exts: + non_py_files.extend([str(f) for f in p.glob(f"**/*.{ext}")]) + + args = ["ruff", "check", "--force-exclude"] + paths + non_py_files + + if config["exclude"]: + args.append(f"--extend-exclude={','.join(config['exclude'])}") + + if lintargs.get("fix"): + # Do a first pass with --fix-only as the json format doesn't return the + # number of fixed issues. + output = run_process(config, args + ["--fix-only"]) + matches = re.match(r"Fixed (\d+) errors?.", output) + if matches: + fixed = int(matches[1]) + + output = run_process(config, args + ["--format=json"]) + if not output: + return [] + + try: + issues = json.loads(output) + except json.JSONDecodeError: + log.error(f"could not parse output: {output}") + return [] + + warning_rules = set(config.get("warning-rules", [])) + for issue in issues: + res = { + "path": issue["filename"], + "lineno": issue["location"]["row"], + "column": issue["location"]["column"], + "lineoffset": issue["end_location"]["row"] - issue["location"]["row"], + "message": issue["message"], + "rule": issue["code"], + "level": "warning" if issue["code"] in warning_rules else "error", + } + if issue["fix"]: + res["hint"] = issue["fix"]["message"] + + results.append(result.from_config(config, **res)) + + return {"results": results, "fixed": fixed} diff --git a/tools/lint/python/ruff_requirements.in b/tools/lint/python/ruff_requirements.in new file mode 100644 index 000000000000..af3ee5763862 --- /dev/null +++ b/tools/lint/python/ruff_requirements.in @@ -0,0 +1 @@ +ruff diff --git a/tools/lint/python/ruff_requirements.txt b/tools/lint/python/ruff_requirements.txt new file mode 100644 index 000000000000..1c8943c26b69 --- /dev/null +++ b/tools/lint/python/ruff_requirements.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: +# +# pip-compile --generate-hashes ruff_requirements.in +# +ruff==0.0.254 \ + --hash=sha256:059a380c08e849b6f312479b18cc63bba2808cff749ad71555f61dd930e3c9a2 \ + --hash=sha256:09c764bc2bd80c974f7ce1f73a46092c286085355a5711126af351b9ae4bea0c \ + --hash=sha256:0deb1d7226ea9da9b18881736d2d96accfa7f328c67b7410478cc064ad1fa6aa \ + --hash=sha256:0eb66c9520151d3bd950ea43b3a088618a8e4e10a5014a72687881e6f3606312 \ + --hash=sha256:27d39d697fdd7df1f2a32c1063756ee269ad8d5345c471ee3ca450636d56e8c6 \ + --hash=sha256:2fc21d060a3197ac463596a97d9b5db2d429395938b270ded61dd60f0e57eb21 \ + --hash=sha256:688379050ae05394a6f9f9c8471587fd5dcf22149bd4304a4ede233cc4ef89a1 \ + --hash=sha256:8deba44fd563361c488dedec90dc330763ee0c01ba54e17df54ef5820079e7e0 \ + --hash=sha256:ac1429be6d8bd3db0bf5becac3a38bd56f8421447790c50599cd90fd53417ec4 \ + --hash=sha256:b3f15d5d033fd3dcb85d982d6828ddab94134686fac2c02c13a8822aa03e1321 \ + --hash=sha256:b435afc4d65591399eaf4b2af86e441a71563a2091c386cadf33eaa11064dc09 \ + --hash=sha256:c38291bda4c7b40b659e8952167f386e86ec29053ad2f733968ff1d78b4c7e15 \ + --hash=sha256:d4385cdd30153b7aa1d8f75dfd1ae30d49c918ead7de07e69b7eadf0d5538a1f \ + --hash=sha256:dd58c500d039fb381af8d861ef456c3e94fd6855c3d267d6c6718c9a9fe07be0 \ + --hash=sha256:e15742df0f9a3615fbdc1ee9a243467e97e75bf88f86d363eee1ed42cedab1ec \ + --hash=sha256:ef20bf798ffe634090ad3dc2e8aa6a055f08c448810a2f800ab716cc18b80107 \ + --hash=sha256:f70dc93bc9db15cccf2ed2a831938919e3e630993eeea6aba5c84bc274237885 + # via -r ruff_requirements.in diff --git a/tools/lint/ruff.yml b/tools/lint/ruff.yml new file mode 100644 index 000000000000..b05c347c0cba --- /dev/null +++ b/tools/lint/ruff.yml @@ -0,0 +1,17 @@ +--- +ruff: + description: An extremely fast Python linter, written in Rust + # Excludes should be added to topsrcdir/pyproject.toml + exclude: [] + # The configure option is used by the build system + extensions: ["configure", "py"] + support-files: + - "**/.ruff.toml" + - "**/ruff.toml" + - "**/pyproject.toml" + - "tools/lint/python/ruff.py" + # Rules that should result in warnings rather than errors. + warning-rules: [] + type: external + payload: python.ruff:lint + setup: python.ruff:setup diff --git a/tools/lint/test/files/flake8/.flake8 b/tools/lint/test/files/flake8/.flake8 deleted file mode 100644 index 1933432319ea..000000000000 --- a/tools/lint/test/files/flake8/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 100 -exclude = - subdir/exclude, diff --git a/tools/lint/test/files/flake8/bad.py b/tools/lint/test/files/flake8/bad.py deleted file mode 100644 index 9d9751c7eb97..000000000000 --- a/tools/lint/test/files/flake8/bad.py +++ /dev/null @@ -1,5 +0,0 @@ -# Unused import -import distutils - -print("This is a line that is over 80 characters but under 100. It shouldn't fail.") -print("This is a line that is over not only 80, but 100 characters. It should most certainly cause a failure.") diff --git a/tools/lint/test/files/flake8/custom/.flake8 b/tools/lint/test/files/flake8/custom/.flake8 deleted file mode 100644 index cfe68833f239..000000000000 --- a/tools/lint/test/files/flake8/custom/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length=110 -ignore= - F401 diff --git a/tools/lint/test/files/flake8/custom/good.py b/tools/lint/test/files/flake8/custom/good.py deleted file mode 100644 index 7f9121a2ba7a..000000000000 --- a/tools/lint/test/files/flake8/custom/good.py +++ /dev/null @@ -1,5 +0,0 @@ -# Unused import -import distutils - -print("This is a line that is over 80 characters but under 100. It shouldn't fail.") -print("This is a line that is over not only 80, but 100 characters. It should also not cause a failure.") diff --git a/tools/lint/test/files/flake8/ext/bad.configure b/tools/lint/test/files/flake8/ext/bad.configure deleted file mode 100644 index 8214ebb3c06d..000000000000 --- a/tools/lint/test/files/flake8/ext/bad.configure +++ /dev/null @@ -1,2 +0,0 @@ -# unused import -import os diff --git a/tools/lint/test/files/flake8/subdir/exclude/bad.py b/tools/lint/test/files/flake8/subdir/exclude/bad.py deleted file mode 100644 index 9d9751c7eb97..000000000000 --- a/tools/lint/test/files/flake8/subdir/exclude/bad.py +++ /dev/null @@ -1,5 +0,0 @@ -# Unused import -import distutils - -print("This is a line that is over 80 characters but under 100. It shouldn't fail.") -print("This is a line that is over not only 80, but 100 characters. It should most certainly cause a failure.") diff --git a/tools/lint/test/files/flake8/subdir/exclude/exclude_subdir/bad.py b/tools/lint/test/files/flake8/subdir/exclude/exclude_subdir/bad.py deleted file mode 100644 index 9d9751c7eb97..000000000000 --- a/tools/lint/test/files/flake8/subdir/exclude/exclude_subdir/bad.py +++ /dev/null @@ -1,5 +0,0 @@ -# Unused import -import distutils - -print("This is a line that is over 80 characters but under 100. It shouldn't fail.") -print("This is a line that is over not only 80, but 100 characters. It should most certainly cause a failure.") diff --git a/tools/lint/test/files/ruff/bad.py b/tools/lint/test/files/ruff/bad.py new file mode 100644 index 000000000000..0015d7e7f9d6 --- /dev/null +++ b/tools/lint/test/files/ruff/bad.py @@ -0,0 +1,4 @@ +import distutils + +if not "foo" in "foobar": + print("oh no!") diff --git a/tools/lint/test/files/ruff/ruff.toml b/tools/lint/test/files/ruff/ruff.toml new file mode 100644 index 000000000000..34f5ca74a453 --- /dev/null +++ b/tools/lint/test/files/ruff/ruff.toml @@ -0,0 +1 @@ +# Empty config to force ruff to ignore the global one. diff --git a/tools/lint/test/python.ini b/tools/lint/test/python.ini index c09db40dccf4..4d181380a180 100644 --- a/tools/lint/test/python.ini +++ b/tools/lint/test/python.ini @@ -13,8 +13,6 @@ skip-if = os == "win" # busts the tree for subsequent tasks on the same worker [test_file_perm.py] skip-if = os == "win" [test_file_whitespace.py] -[test_flake8.py] -requirements = tools/lint/python/flake8_requirements.txt [test_fluent_lint.py] [test_lintpref.py] [test_manifest_alpha.py] @@ -26,6 +24,8 @@ requirements = tools/lint/python/flake8_requirements.txt requirements = tools/lint/python/pylint_requirements.txt [test_rst.py] requirements = tools/lint/rst/requirements.txt +[test_ruff.py] +requirements = tools/lint/python/ruff_requirements.txt [test_rustfmt.py] [test_shellcheck.py] [test_trojan_source.py] diff --git a/tools/lint/test/test_flake8.py b/tools/lint/test/test_flake8.py deleted file mode 100644 index d44e3828edd2..000000000000 --- a/tools/lint/test/test_flake8.py +++ /dev/null @@ -1,117 +0,0 @@ -import os - -import mozunit - -LINTER = "flake8" -fixed = 0 - - -def test_lint_single_file(lint, paths): - results = lint(paths("bad.py")) - assert len(results) == 2 - assert results[0].rule == "F401" - assert results[0].level == "error" - assert results[1].rule == "E501" - assert results[1].level == "error" - assert results[1].lineno == 5 - - # run lint again to make sure the previous results aren't counted twice - results = lint(paths("bad.py")) - assert len(results) == 2 - - -def test_lint_custom_config_ignored(lint, paths): - results = lint(paths("custom")) - assert len(results) == 2 - - results = lint(paths("custom/good.py")) - assert len(results) == 2 - - -def test_lint_fix(lint, create_temp_file): - global fixed - contents = """ -import distutils - -def foobar(): - pass -""".lstrip() - - path = create_temp_file(contents, name="bad.py") - results = lint([path]) - assert len(results) == 2 - - # Make sure the missing blank line is fixed, but the unused import isn't. - results = lint([path], fix=True) - assert len(results) == 1 - assert fixed == 1 - - fixed = 0 - - # Also test with a directory - path = os.path.dirname(create_temp_file(contents, name="bad2.py")) - results = lint([path], fix=True) - # There should now be two files with 2 combined errors - assert len(results) == 2 - assert fixed == 1 - assert all(r.rule != "E501" for r in results) - - -def test_lint_fix_uses_config(lint, create_temp_file): - contents = """ -foo = ['A list of strings', 'that go over 80 characters', 'to test if autopep8 fixes it'] -""".lstrip() - - path = create_temp_file(contents, name="line_length.py") - lint([path], fix=True) - - # Make sure autopep8 reads the global config under lintargs['root']. If it - # didn't, then the line-length over 80 would get fixed. - with open(path, "r") as fh: - assert fh.read() == contents - - -def test_lint_excluded_file(lint, paths, config): - # First file is globally excluded, second one is from .flake8 config. - files = paths("bad.py", "subdir/exclude/bad.py", "subdir/exclude/exclude_subdir") - config["exclude"] = paths("bad.py") - results = lint(files, config) - print(results) - assert len(results) == 0 - - # Make sure excludes also apply when running from a different cwd. - cwd = paths("subdir")[0] - os.chdir(cwd) - - results = lint(paths("subdir/exclude")) - print(results) - assert len(results) == 0 - - -def test_lint_excluded_file_with_glob(lint, paths, config): - config["exclude"] = paths("ext/*.configure") - - files = paths("ext") - results = lint(files, config) - print(results) - assert len(results) == 0 - - files = paths("ext/bad.configure") - results = lint(files, config) - print(results) - assert len(results) == 0 - - -def test_lint_excluded_file_with_no_filter(lint, paths, config): - results = lint(paths("subdir/exclude"), use_filters=False) - print(results) - assert len(results) == 4 - - -def test_lint_uses_custom_extensions(lint, paths): - assert len(lint(paths("ext"))) == 1 - assert len(lint(paths("ext/bad.configure"))) == 1 - - -if __name__ == "__main__": - mozunit.main() diff --git a/tools/lint/test/test_ruff.py b/tools/lint/test/test_ruff.py new file mode 100644 index 000000000000..fbb483780e54 --- /dev/null +++ b/tools/lint/test/test_ruff.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pprint import pprint +from textwrap import dedent + +import mozunit + +LINTER = "ruff" +fixed = 0 + + +def test_lint_fix(lint, create_temp_file): + contents = dedent( + """ + import distutils + print("hello!") + """ + ) + + path = create_temp_file(contents, "bad.py") + lint([path], fix=True) + assert fixed == 1 + + +def test_lint_ruff(lint, paths): + results = lint(paths()) + pprint(results, indent=2) + assert len(results) == 2 + assert results[0].level == "error" + assert results[0].relpath == "bad.py" + assert "`distutils` imported but unused" in results[0].message + + +if __name__ == "__main__": + mozunit.main()