Bug 1879254 - Verify .gitignore and .hgignore consistency. r=glandium

Differential Revision: https://phabricator.services.mozilla.com/D201072

.gitignore

@@ -1,27 +1,43 @@
 # .gitignore - List of filenames git should ignore
 
+# See docs/code-quality/lint/linters/ignorefile.rst for lint-ignore-next-line
+# syntax.
+
 # Filenames that should be ignored wherever they appear
 *~
 *.pyc
 *.pyo
 TAGS
 tags
-compile_commands.json
+.DS_Store
-toolchains.json
-
-# Ignore ID generated by idutils and un-ignore id directory (for Indonesian locale)
-ID
-!id/
-.DS_Store*
 *.pdb
 .eslintcache
+*.gcda
+*.gcno
+*.gcov
+compile_commands.json
+
+# Ignore ID generated by idutils.
+ID
+
+# Un-ignore id directory (for Indonesian locale)
+# lint-ignore-next-line: git-only
+!id/
+
+# Generated by hg or patch (e.g. revert, failed patch, ...)
+*.orig
+*.rej
+
 # Filesystem temporaries
 .fuse_hidden*
 
 # Ignore Python .egg-info directories for first-party modules (but,
 # still add vendored packages' .egg-info directories)
+# lint-ignore-next-line: syntax-difference
 *.egg-info
+# lint-ignore-next-line: syntax-difference
 !third_party/python/**/*.egg-info
+# lint-ignore-next-line: syntax-difference
 !testing/web-platform/tests/tools/third_party/**/*.egg-info
 
 # Vim swap files.
@@ -38,12 +54,14 @@ ID
 /.clangd
 /.mozconfig*
 /mozconfig*
+/.moz-fast-forward
 /old-configure
 /config.cache
 /config.log
 /.clang_complete
 /machrc
 /.machrc
+
 # pyenv artifact
 /.python-version
 
@@ -56,10 +74,15 @@ security/manager/.nss.checkout
 # gecko.log is generated by various test harnesses
 /gecko.log
 
-# Ignore all node_modules directories
+# Ignore all node_modules directories except for ones under third_party
 node_modules/
-# ...but allow ones under third_party
+devtools/**/node_modules/
-!/third_party/**/node_modules/
+tools/browsertime/node_modules/
+tools/lint/eslint/eslint-plugin-mozilla/node_modules/
+browser/components/asrouter/node_modules/
+browser/components/newtab/node_modules/
+browser/components/aboutwelcome/node_modules/
+tools/esmify/node_modules/
 
 # Ignore newtab component build assets
 browser/components/newtab/logs/
@@ -91,9 +114,16 @@ toolkit/components/translations/bergamot-translator/thirdparty
 # SpiderMonkey configury
 js/src/old-configure
 js/src/autom4te.cache
+
 # SpiderMonkey test result logs
 js/src/tests/results-*.html
 js/src/tests/results-*.txt
+js/src/devtools/rootAnalysis/t/out
+
+# SpiderMonkey wasm/generate-spectests artifacts
+js/src/jit-test/etc/wasm/generate-spectests/specs/
+js/src/jit-test/etc/wasm/generate-spectests/tests/
+js/src/jit-test/etc/wasm/generate-spectests/target/
 
 # Java HTML5 parser classes
 parser/html/java/htmlparser/
@@ -101,6 +131,9 @@ parser/html/java/javaparser/
 parser/html/java/javaparser.jar
 parser/html/java/translator.jar
 
+# SVN directories
+.svn/
+
 # Ignore the files and directory that Eclipse IDE creates
 .project
 .cproject
@@ -109,6 +142,7 @@ parser/html/java/translator.jar
 # Ignore the files and directory that JetBrains IDEs create.
 /.idea/
 *.iml
+
 # Android Monitor in Android Studio creates a captures/ directory.
 /captures/
 
@@ -126,16 +160,42 @@ devtools/shared/chrome.manifest
 devtools/client/debugger/assets/build
 devtools/client/debugger/assets/module-manifest.json
 
-# Ignore node_modules directories in devtools
+# Ignore node_module directories and npm artifacts
-devtools/**/node_modules
+remote/test/puppeteer/**.tsbuildinfo
+remote/test/puppeteer/**/lib/
+remote/test/puppeteer/**/node_modules/
+remote/test/puppeteer/**/.wireit/
+remote/test/puppeteer/.devcontainer/
+remote/test/puppeteer/.github
+remote/test/puppeteer/.husky
+remote/test/puppeteer/.wireit/
+remote/test/puppeteer/coverage/
+remote/test/puppeteer/docker/
+remote/test/puppeteer/docs/puppeteer-core.api.json
+remote/test/puppeteer/docs/puppeteer.api.json
+remote/test/puppeteer/experimental/
+remote/test/puppeteer/lib/
+remote/test/puppeteer/node_modules/
+remote/test/puppeteer/package-lock.json
+remote/test/puppeteer/packages/ng-schematics/test/build
+remote/test/puppeteer/packages/puppeteer/**/README.md
+remote/test/puppeteer/packages/puppeteer-core/src/generated
+remote/test/puppeteer/packages/puppeteer-core/**/README.md
+remote/test/puppeteer/src/generated
+remote/test/puppeteer/test/build
+remote/test/puppeteer/test/installation/puppeteer*.tgz
+remote/test/puppeteer/test/output-firefox
+remote/test/puppeteer/test/output-chromium
+remote/test/puppeteer/testserver/lib/
+remote/test/puppeteer/tools/internal/
+remote/test/puppeteer/tools/mocha-runner/bin/
+remote/test/puppeteer/website
 
-# Ignore browsertime output directory
+third_party/js/PKI.js/node_modules/
-browsertime-results
+third_party/js/PKI.js/package-lock.json
 
-# Ignore the build directories of WebGPU and WebRender standalone builds.
+# git checkout of libstagefright
-gfx/wgpu/target
+media/libstagefright/android
-gfx/wgpu/.*/build
-gfx/wr/target/
 
 # Tag files generated by GNU Global
 GTAGS
@@ -169,8 +229,8 @@ testing/mozharness/logs/
 testing/mozharness/.coverage
 testing/mozharness/nosetests.xml
 
-# Ignore ESLint node_modules
+# Ignore tox generated dir
-node_modules/
+.tox/
 
 # Ignore talos virtualenv and tp5n files.
 # The tp5n set is supposed to be decompressed at
@@ -183,25 +243,48 @@ testing/talos/bin/
 testing/talos/include/
 testing/talos/lib/
 testing/talos/talos/fis/tp5n.zip
+testing/talos/talos/fis/tp5n.tar.gz
 testing/talos/talos/fis/tp5n
 testing/talos/talos/tests/tp5n.zip
+testing/talos/talos/tests/tp5n.tar.gz
 testing/talos/talos/tests/tp5n
 testing/talos/talos/tests/devtools/damp.manifest.develop
+testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint.manifest.develop
+testing/talos/talos/webextensions/
+talos-venv
+py3venv
+testing/talos/talos/mitmproxy/mitmdump
+testing/talos/talos/mitmproxy/mitmproxy
+testing/talos/talos/mitmproxy/mitmweb
+
+# Ignore talos webkit benchmark files; source is copied from in-tree /third_party
+# into testing/talos/talos/tests/webkit/PerformanceTests/ when run locally
+# i.e. speedometer, motionmark, stylebench
+testing/talos/talos/tests/webkit/PerformanceTests
 
 # Ignore sync tps logs and reports
 tps.log
 tps_result.json
 
+# Ignore toolchains.json created by tooltool.
+toolchains.json
+
 # Unit test
 .pytest_cache/
 
+# Ruff
+.ruff_cache/
+
 # Ignore files created when running a reftest.
 lextab.py
 
 # Ignore Visual Studio/Visual Studio Code workspace files.
 .vs/
+# lint-ignore-next-line: syntax-difference
 .vscode/
+# lint-ignore-next-line: syntax-difference
 !.vscode/extensions.json
+# lint-ignore-next-line: syntax-difference
 !.vscode/tasks.json
 *.user
 
@@ -214,6 +297,20 @@ testing/raptor/raptor-venv
 testing/raptor/raptor/tests/json/
 testing/raptor/webext/raptor/auto_gen_test_config.js
 
+# Ignore condprofile build directory
+testing/condprofile/build
+
+# Ignore browsertime output directory
+browsertime-results
+
+# Ignore the build directories of WebGPU and WebRender standalone builds.
+gfx/wgpu/target
+gfx/wgpu/**/build
+gfx/wr/target/
+
+# Ignore Rust/Cargo output from running `cargo` directly for image_builder docker image
+taskcluster/docker/image_builder/build-image/target
+
 # Ignore ICU4X experimentation data files.
 # See intl/ICU4X.md for more details.
 config/external/icu4x
@@ -221,6 +318,9 @@ config/external/icu4x
 # Ignore the index files generated by clangd.
 .cache/clangd/index/
 
+# Ignore mypy files
+.mypy_cache/
+
 # Ignore Storybook generated files
 browser/components/storybook/node_modules/
 browser/components/storybook/storybook-static/
@@ -233,5 +333,13 @@ tools/esmify/jscodeshift.cmd
 tools/esmify/jscodeshift.ps1
 tools/esmify/package-lock.json
 
+# Ignore support files for en-US dictionary updates
+extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/scowl
+extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/support_files/
+extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/*en_US-mozilla*
+
 # Ignore automatically generated mots documentation
 docs/mots/index.rst
+
+# Ignore generated directory with .class files for GeckoView annotation processor
+mobile/android/annotations/bin/

.hgignore

@@ -1,30 +1,40 @@
 # .hgignore - List of filenames hg should ignore
 
+# See docs/code-quality/lint/linters/ignorefile.rst for lint-ignore-next-line
+# syntax.
+
 # Filenames that should be ignored wherever they appear
 ~$
-\.py(c|o)$
+\.pyc$
+\.pyo$
 (^|/)TAGS$
 (^|/)tags$
-(^|/)ID$
 (^|/)\.DS_Store$
-\.pdb
+\.pdb$
-\.eslintcache
+(^|/)\.eslintcache$
-\.gcda
+\.gcda$
-\.gcno
+\.gcno$
-\.gcov
+\.gcov$
-compile_commands\.json
+(^|/)compile_commands\.json$
 
-# Generated by hg (e.g. revert, failed patch, ...)
+# Ignore ID generated by idutils.
+(^|/)ID$
+
+# Generated by hg or patch (e.g. revert, failed patch, ...)
 \.orig$
 \.rej$
 
+# Filesystem temporaries
+(^|/)\.fuse_hidden.*$
+
 # Ignore Python .egg-info directories for first-party modules (but,
 # still add vendored packages' .egg-info directories)
+# lint-ignore-next-line: syntax-difference
 ^(?=.*\.egg-info/)(?!^third_party/python/)(?!^testing/web-platform/tests/tools/third_party/)
 
 # Vim swap files.
-^\.sw.$
+^\.sw[a-z]$
-.[^/]*\.sw.$
+\.[^/]*\.sw[a-z]$
 
 # Emacs directory variable files.
 \.dir-locals\.el
@@ -41,7 +51,9 @@ compile_commands\.json
 ^config\.cache$
 ^config\.log$
 ^\.clang_complete
-^\.?machrc$
+^machrc$
+^\.machrc$
+
 # pyenv artifact
 ^\.python-version$
 
@@ -49,31 +61,41 @@ compile_commands\.json
 ^security/manager/\.nss\.checkout$
 
 # Build directories
-^obj
+^obj.*/
 
 # gecko.log is generated by various test harnesses
-^gecko\.log
+^gecko\.log$
+
+# Ignore all node_modules directories except for ones under third_party
+^node_modules/
+^devtools/.*/node_modules/
+^tools/browsertime/node_modules/
+^tools/lint/eslint/eslint-plugin-mozilla/node_modules/
+^browser/components/asrouter/node_modules/
+^browser/components/newtab/node_modules/
+^browser/components/aboutwelcome/node_modules/
+^tools/esmify/node_modules/
 
 # Ignore newtab component build assets
 ^browser/components/newtab/logs/
 
-# Ignore about:welcome build assets
+# Ignore about:welcome component build assets
 ^browser/components/aboutwelcome/logs/
 
 # Ignore ASRouter component build assets
 ^browser/components/asrouter/logs/
 
 # Ignore ASRouter generated test files
-^browser/components/newtab/content-src/asrouter/schemas/corpus/CFRMessageProvider.messages.json
+^browser/components/newtab/content-src/asrouter/schemas/corpus/CFRMessageProvider\.messages\.json
-^browser/components/newtab/content-src/asrouter/schemas/corpus/OnboardingMessageProvider.messages.json
+^browser/components/newtab/content-src/asrouter/schemas/corpus/OnboardingMessageProvider\.messages\.json
-^browser/components/newtab/content-src/asrouter/schemas/corpus/PanelTestProvider.messages.json
+^browser/components/newtab/content-src/asrouter/schemas/corpus/PanelTestProvider\.messages\.json
-^browser/components/newtab/content-src/asrouter/schemas/corpus/PanelTestProvider_toast_notification.messages.json
+^browser/components/newtab/content-src/asrouter/schemas/corpus/PanelTestProvider_toast_notification\.messages\.json
 
 # Ignore Pocket component build and dev assets
-browser/components/pocket/content/panels/css/main.compiled.css.map
+^browser/components/pocket/content/panels/css/main\.compiled\.css\.map
 
 # Ignore downloaded thirdparty build artifacts.
-toolkit/components/translations/bergamot-translator/thirdparty
+^toolkit/components/translations/bergamot-translator/thirdparty
 
 # Build directories for js shell
 _DBG\.OBJ/
@@ -83,17 +105,23 @@ _OPT\.OBJ/
 
 # SpiderMonkey configury
 ^js/src/old-configure$
-^js/src/autom4te.cache$
+^js/src/autom4te\.cache$
+
 # SpiderMonkey test result logs
-^js/src/tests/results-.*\.(html|txt)$
+^js/src/tests/results-.*\.html$
+^js/src/tests/results-.*\.txt$
 ^js/src/devtools/rootAnalysis/t/out
+
 # SpiderMonkey wasm/generate-spectests artifacts
 ^js/src/jit-test/etc/wasm/generate-spectests/specs/
 ^js/src/jit-test/etc/wasm/generate-spectests/tests/
 ^js/src/jit-test/etc/wasm/generate-spectests/target/
 
 # Java HTML5 parser classes
-^parser/html/java/(html|java)parser/
+^parser/html/java/htmlparser/
+^parser/html/java/javaparser/
+^parser/html/java/javaparser\.jar$
+^parser/html/java/translator\.jar$
 
 # SVN directories
 \.svn/
@@ -106,24 +134,27 @@ _OPT\.OBJ/
 # Ignore the files and directory that JetBrains IDEs create.
 \.idea/
 \.iml$
+
 # Android Monitor in Android Studio creates a captures/ directory.
 ^captures/
 
 # Gradle cache.
-^.gradle/
+^\.gradle/
 
 # Local Gradle configuration properties.
-^local.properties$
+^local\.properties$
 
 # Git repositories
-.git/
+# lint-ignore-next-line: hg-only
+\.git/
 
 # Ignore chrome.manifest files from the devtools loader
-^devtools/client/chrome.manifest$
+^devtools/client/chrome\.manifest$
-^devtools/shared/chrome.manifest$
+^devtools/shared/chrome\.manifest$
 
-# Ignore node_modules directories in devtools
+# Ignore debugger build directories
-^devtools/.*/node_modules/
+^devtools/client/debugger/assets/build
+^devtools/client/debugger/assets/module-manifest\.json
 
 # Ignore node_module directories and npm artifacts
 ^remote/test/puppeteer/.*\.tsbuildinfo
@@ -143,9 +174,9 @@ _OPT\.OBJ/
 ^remote/test/puppeteer/node_modules/
 ^remote/test/puppeteer/package-lock\.json
 ^remote/test/puppeteer/packages/ng-schematics/test/build
-^remote/test/puppeteer/packages/puppeteer/.*/README.md
+^remote/test/puppeteer/packages/puppeteer/.*/README\.md
 ^remote/test/puppeteer/packages/puppeteer-core/src/generated
-^remote/test/puppeteer/packages/puppeteer-core/.*/README.md
+^remote/test/puppeteer/packages/puppeteer-core/.*/README\.md
 ^remote/test/puppeteer/src/generated
 ^remote/test/puppeteer/test/build
 ^remote/test/puppeteer/test/installation/puppeteer.*\.tgz
@@ -156,8 +187,8 @@ _OPT\.OBJ/
 ^remote/test/puppeteer/tools/mocha-runner/bin/
 ^remote/test/puppeteer/website
 
-^third_party/js/PKI.js/node_modules/
+^third_party/js/PKI\.js/node_modules/
-^third_party/js/PKI.js/package-lock.json
+^third_party/js/PKI\.js/package-lock\.json
 
 # git checkout of libstagefright
 ^media/libstagefright/android$
@@ -175,7 +206,7 @@ _OPT\.OBJ/
 ^testing/web-platform/products/
 
 # Android Gradle artifacts.
-^mobile/android/gradle/.gradle
+^mobile/android/gradle/\.gradle
 
 # XCode project cruft
 ^[^/]*\.xcodeproj/
@@ -188,23 +219,14 @@ _OPT\.OBJ/
 ^dom/webgpu/tests/cts/vendor/target/
 
 # Ignore mozharness execution files
-^testing/mozharness/.tox/
+^testing/mozharness/\.tox/
 ^testing/mozharness/build/
 ^testing/mozharness/logs/
-^testing/mozharness/.coverage
+^testing/mozharness/\.coverage
-^testing/mozharness/nosetests.xml
+^testing/mozharness/nosetests\.xml
 
 # Ignore tox generated dir
-.tox/
+\.tox/
-
-# Ignore ESLint and other tool's node_modules.
-^node_modules/
-^tools/browsertime/node_modules/
-^tools/lint/eslint/eslint-plugin-mozilla/node_modules/
-^browser/components/asrouter/node_modules/
-^browser/components/newtab/node_modules/
-^browser/components/aboutwelcome/node_modules/
-^tools/esmify/node_modules/
 
 # Ignore talos virtualenv and tp5n files.
 # The tp5n set is supposed to be decompressed at
@@ -212,18 +234,18 @@ _OPT\.OBJ/
 # locally. Similarly, running talos requires a Python package virtual
 # environment. Both the virtual environment and tp5n files end up littering
 # the status command, so we ignore them.
-^testing/talos/.Python
+^testing/talos/\.Python
 ^testing/talos/bin/
 ^testing/talos/include/
 ^testing/talos/lib/
-^testing/talos/talos/fis/tp5n.zip
+^testing/talos/talos/fis/tp5n\.zip
-^testing/talos/talos/fis/tp5n.tar.gz
+^testing/talos/talos/fis/tp5n\.tar\.gz
 ^testing/talos/talos/fis/tp5n
-^testing/talos/talos/tests/tp5n.zip
+^testing/talos/talos/tests/tp5n\.zip
-^testing/talos/talos/tests/tp5n.tar.gz
+^testing/talos/talos/tests/tp5n\.tar\.gz
 ^testing/talos/talos/tests/tp5n
-^testing/talos/talos/tests/devtools/damp.manifest.develop
+^testing/talos/talos/tests/devtools/damp\.manifest\.develop
-^testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint.manifest.develop
+^testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint\.manifest\.develop
 ^testing/talos/talos/webextensions/
 ^talos-venv
 ^py3venv
@@ -236,8 +258,12 @@ _OPT\.OBJ/
 # i.e. speedometer, motionmark, stylebench
 ^testing/talos/talos/tests/webkit/PerformanceTests
 
+# Ignore sync tps logs and reports
+tps\.log$
+tps_result\.json$
+
 # Ignore toolchains.json created by tooltool.
-^toolchains\.json
+^toolchains\.json$
 
 # Unit test
 \.pytest_cache/
@@ -246,25 +272,22 @@ _OPT\.OBJ/
 \.ruff_cache/
 
 # Ignore files created when running a reftest.
-^lextab.py$
+^lextab\.py$
-
-# Ignore sync tps logs and reports
-tps\.log
-tps_result\.json
 
 # Ignore Visual Studio/Visual Studio Code workspace files.
-\.vs/
+^\.vs/
-\.vscode/(?!extensions\.json|tasks\.json)
+# lint-ignore-next-line: syntax-difference
-\.user$
+^\.vscode/(?!extensions\.json$|tasks\.json$)
+.*\.user$
 
-# https://bz.mercurial-scm.org/show_bug.cgi?id=5322
+# Thunderbird source tree
 ^comm/
 
 # Ignore various raptor performance framework files
-^testing/raptor/.raptor-venv
+^testing/raptor/\.raptor-venv
 ^testing/raptor/raptor-venv
 ^testing/raptor/raptor/tests/json/
-^testing/raptor/webext/raptor/auto_gen_test_config.js
+^testing/raptor/webext/raptor/auto_gen_test_config\.js
 
 # Ignore condprofile build directory
 ^testing/condprofile/build
@@ -273,9 +296,9 @@ tps_result\.json
 ^browsertime-results
 
 # Ignore the build directories of WebGPU and WebRender standalone builds.
-gfx/wgpu/target
+^gfx/wgpu/target
-gfx/wgpu/.*/build
+^gfx/wgpu/.*/build
-gfx/wr/target/
+^gfx/wr/target/
 
 # Ignore Rust/Cargo output from running `cargo` directly for image_builder docker image
 ^taskcluster/docker/image_builder/build-image/target
@@ -285,7 +308,7 @@ gfx/wr/target/
 ^config/external/icu4x
 
 # Ignore the index files generated by clangd.
-^.cache/clangd/index/
+^\.cache/clangd/index/
 
 # Ignore mypy files
 \.mypy_cache/
@@ -293,22 +316,22 @@ gfx/wr/target/
 # Ignore Storybook generated files
 ^browser/components/storybook/node_modules/
 ^browser/components/storybook/storybook-static/
-^browser/components/storybook/.storybook/chrome-map.js
+^browser/components/storybook/\.storybook/chrome-map\.js
-^browser/components/storybook/custom-elements.json
+^browser/components/storybook/custom-elements\.json
 
 # Ignore jscodeshift installed by mach esmify on windows
 ^tools/esmify/jscodeshift
-^tools/esmify/jscodeshift.cmd
+^tools/esmify/jscodeshift\.cmd
-^tools/esmify/jscodeshift.ps1
+^tools/esmify/jscodeshift\.ps1
-^tools/esmify/package-lock.json
+^tools/esmify/package-lock\.json
 
 # Ignore support files for en-US dictionary updates
 ^extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/scowl
 ^extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/support_files/
-^extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/*en_US-mozilla*
+^extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/.*en_US-mozilla.*
 
 # Ignore automatically generated mots documentation
-^docs/mots/index.rst
+^docs/mots/index\.rst
 
 # Ignore generated directory with .class files for GeckoView annotation processor
 ^mobile/android/annotations/bin/

docs/code-quality/lint/linters/ignorefile.rst

@@ -0,0 +1,45 @@
+Ignorefile Lint
+===============
+
+Ignorefile lint is a linter for ``.gitignore`` and ``.hgignore`` files,
+to verify those files have equivalent entries.
+
+Each pattern is roughly compared, ignoring punctuations, to absorb the
+syntax difference.
+
+Run Locally
+-----------
+
+The mozlint integration of ignorefile linter can be run using mach:
+
+.. parsed-literal::
+
+    $ mach lint --linter ignorefile
+
+
+Special syntax
+--------------
+
+The following special comment can be used to ignore the pattern in the next line.
+
+.. parsed-literal::
+
+    # lint-ignore-next-line: git-only
+    # lint-ignore-next-line: hg-only
+
+The next line exists only in ``.gitignore``. or ``.hgignore``.
+
+.. parsed-literal::
+    # lint-ignore-next-line: syntax-difference
+
+The next line's pattern needs to be represented differently between
+``.gitignore`` and ``.hgignore``.
+This can be used when the ``.hgignore`` uses complex pattern which cannot be
+represented in single pattern in ``.gitignore``.
+
+
+Sources
+-------
+
+* :searchfox:`Configuration (YAML) <tools/lint/ignorefile.yml>`
+* :searchfox:`Source <tools/lint/ignorefile/__init__.py>`

taskcluster/ci/source-test/mozlint.yml

@@ -607,3 +607,15 @@ trojan-source:
             - '**/*.py'
             - '**/*.rs'
             - 'tools/lint/trojan-source.yml'
+
+ignorefile-test:
+    description: .gitignore vs .hgignore consistency check
+    platform: lint/opt
+    treeherder:
+        symbol: ignorefile
+    run:
+        mach: lint -v -l ignorefile .gitignore .hgignore
+    when:
+        files-changed:
+            - '.gitignore'
+            - '.hgignore'

tools/lint/ignorefile.yml

@@ -0,0 +1,10 @@
+---
+ignorefile:
+    description: Linter for .gitignore and .hgignore files
+    include:
+        - '.gitignore'
+        - '.hgignore'
+    support-files:
+        - 'tools/lint/ignorefile**'
+    type: external
+    payload: ignorefile:lint

tools/lint/ignorefile/__init__.py

@@ -0,0 +1,162 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import pathlib
+import re
+
+from mozlint import result
+from mozlint.pathutils import expand_exclusions
+
+validReasons = [
+    "git-only",
+    "hg-only",
+    "syntax-difference",
+]
+
+
+class IgnorePattern:
+    """A class that represents single pattern in .gitignore or .hgignore, and
+    provides rough comparison, and also hashing for SequenceMatcher."""
+
+    def __init__(self, pattern, lineno):
+        self.pattern = pattern
+        self.lineno = lineno
+
+        self.simplePattern = self.removePunctuation(pattern)
+        self.hash = hash(self.pattern)
+
+    @staticmethod
+    def removePunctuation(s):
+        # Remove special characters.
+        # '.' is also removed because .hgignore uses regular expression.
+        s = re.sub(r"[^A-Za-z0-9_~/]", "", s)
+
+        # '/' is kept, except for the following cases:
+        #   * leading '/' in .gitignore to specify top-level file,
+        #     which is represented as '^' in .hgignore
+        #   * leading '(^|/)' in .hgignore to specify filename
+        #     (characters except for '/' are removed above)
+        #   * leading '[^/]*' in .hgignore
+        #     (characters except for '/' are removed above)
+        s = re.sub(r"^/", "", s)
+
+        return s
+
+    def __eq__(self, other):
+        return self.simplePattern == other.simplePattern
+
+    def __hash__(self):
+        return self.hash
+
+
+def parseFile(results, path, config):
+    patterns = []
+    ignoreNextLine = False
+
+    lineno = 0
+    with open(path, "r") as f:
+        for line in f:
+            line = line.rstrip("\n")
+            lineno += 1
+
+            if ignoreNextLine:
+                ignoreNextLine = False
+                continue
+
+            m = re.match(r"^# lint-ignore-next-line: (.+)", line)
+            if m:
+                reason = m.group(1)
+                if reason not in validReasons:
+                    res = {
+                        "path": str(path),
+                        "lineno": lineno,
+                        "message": f'Unknown lint rule: "{reason}"',
+                        "level": "error",
+                    }
+                    results.append(result.from_config(config, **res))
+                    continue
+
+                ignoreNextLine = True
+                continue
+
+            m = line.startswith("#")
+            if m:
+                continue
+
+            if line == "":
+                continue
+
+            patterns.append(IgnorePattern(line, lineno))
+
+    return patterns
+
+
+def getLineno(patterns, index):
+    if index >= len(patterns):
+        return patterns[-1].lineno
+
+    return patterns[index].lineno
+
+
+def doLint(results, path1, config):
+    if path1.name == ".gitignore":
+        path2 = path1.parent / ".hgignore"
+    elif path1.name == ".hgignore":
+        path2 = path1.parent / ".gitignore"
+    else:
+        res = {
+            "path": str(path1),
+            "lineno": 0,
+            "message": "Unsupported file",
+            "level": "error",
+        }
+        results.append(result.from_config(config, **res))
+        return
+
+    patterns1 = parseFile(results, path1, config)
+    patterns2 = parseFile(results, path2, config)
+
+    # Comparison for each line is done via IgnorePattern.__eq__, which
+    # ignores punctuations.
+    if patterns1 == patterns2:
+        return
+
+    # Report minimal differences.
+    from difflib import SequenceMatcher
+
+    s = SequenceMatcher(None, patterns1, patterns2)
+    for tag, index1, _, index2, _ in s.get_opcodes():
+        if tag == "replace":
+            res = {
+                "path": str(path1),
+                "lineno": getLineno(patterns1, index1),
+                "message": f'Pattern mismatch: "{patterns1[index1].pattern}" in {path1.name} vs "{patterns2[index2].pattern}" in {path2.name}',
+                "level": "error",
+            }
+            results.append(result.from_config(config, **res))
+        elif tag == "delete":
+            res = {
+                "path": str(path1),
+                "lineno": getLineno(patterns1, index1),
+                "message": f'Pattern "{patterns1[index1].pattern}" not found in {path2.name}',
+                "level": "error",
+            }
+            results.append(result.from_config(config, **res))
+        elif tag == "insert":
+            res = {
+                "path": str(path1),
+                "lineno": getLineno(patterns1, index1),
+                "message": f'Pattern "{patterns2[index2].pattern}" not found in {path1.name}',
+                "level": "error",
+            }
+            results.append(result.from_config(config, **res))
+
+
+def lint(paths, config, fix=None, **lintargs):
+    results = []
+
+    for path in expand_exclusions(paths, config, lintargs["root"]):
+        doLint(results, pathlib.Path(path), config)
+
+    return {"results": results, "fixed": 0}

tools/lint/test/python.toml

@@ -26,6 +26,8 @@ skip-if = ["os == 'win'"]
 
 ["test_fluent_lint.py"]
 
+["test_ignorefile.py"]
+
 ["test_lintpref.py"]
 
 ["test_manifest_alpha.py"]

tools/lint/test/test_ignorefile.py

@@ -0,0 +1,170 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozunit
+
+LINTER = "ignorefile"
+
+
+def test_same(lint, create_temp_file):
+    hgContents = """
+\\.pyc$
+\\.pyo$
+    """
+    gitContents = """
+*.pyc
+*.pyo
+    """
+
+    path = create_temp_file(hgContents, ".hgignore")
+    create_temp_file(gitContents, ".gitignore")
+
+    results = lint([path])
+
+    assert len(results) == 0
+
+
+def test_replace(lint, create_temp_file):
+    hgContents = """
+\\.pyc$
+\\.pyo$
+    """
+    gitContents = """
+*.pyc
+*.pyp
+    """
+
+    path = create_temp_file(hgContents, ".hgignore")
+    path2 = create_temp_file(gitContents, ".gitignore")
+
+    results = lint([path])
+
+    assert len(results) == 1
+    assert results[0].level == "error"
+    assert results[0].lineno == 3
+    assert (
+        results[0].message
+        == 'Pattern mismatch: "\\.pyo$" in .hgignore vs "*.pyp" in .gitignore'
+    )
+
+    results = lint([path2])
+
+    assert len(results) == 1
+    assert results[0].level == "error"
+    assert results[0].lineno == 3
+    assert (
+        results[0].message
+        == 'Pattern mismatch: "*.pyp" in .gitignore vs "\\.pyo$" in .hgignore'
+    )
+
+
+def test_insert(lint, create_temp_file):
+    hgContents = """
+\\.pyc$
+\\.pyo$
+    """
+    gitContents = """
+*.pyc
+foo
+*.pyo
+    """
+
+    path = create_temp_file(hgContents, ".hgignore")
+    create_temp_file(gitContents, ".gitignore")
+
+    results = lint([path])
+
+    assert len(results) == 1
+    assert results[0].level == "error"
+    assert results[0].lineno == 3
+    assert results[0].message == 'Pattern "foo" not found in .hgignore'
+
+
+def test_delete(lint, create_temp_file):
+    hgContents = """
+\\.pyc$
+foo
+\\.pyo$
+    """
+    gitContents = """
+*.pyc
+*.pyo
+    """
+
+    path = create_temp_file(hgContents, ".hgignore")
+    create_temp_file(gitContents, ".gitignore")
+
+    results = lint([path])
+
+    assert len(results) == 1
+    assert results[0].level == "error"
+    assert results[0].lineno == 3
+    assert results[0].message == 'Pattern "foo" not found in .gitignore'
+
+
+def test_ignore(lint, create_temp_file):
+    hgContents = """
+\\.pyc$
+# lint-ignore-next-line: hg-only
+foo
+\\.pyo$
+# lint-ignore-next-line: syntax-difference
+(file1|file2)
+diff1
+    """
+    gitContents = """
+*.pyc
+*.pyo
+# lint-ignore-next-line: git-only
+bar
+# lint-ignore-next-line: syntax-difference
+file1
+# lint-ignore-next-line: syntax-difference
+file2
+diff2
+    """
+
+    path = create_temp_file(hgContents, ".hgignore")
+    create_temp_file(gitContents, ".gitignore")
+
+    results = lint([path])
+
+    # Only the line without lint-ignore-next-line should be reported
+    assert len(results) == 1
+    assert results[0].level == "error"
+    assert results[0].lineno == 8
+    assert (
+        results[0].message
+        == 'Pattern mismatch: "diff1" in .hgignore vs "diff2" in .gitignore'
+    )
+
+
+def test_invalid_syntax(lint, create_temp_file):
+    hgContents = """
+\\.pyc$
+# lint-ignore-next-line: random
+foo
+\\.pyo$
+    """
+    gitContents = """
+*.pyc
+*.pyo
+    """
+
+    path = create_temp_file(hgContents, ".hgignore")
+    create_temp_file(gitContents, ".gitignore")
+
+    results = lint([path])
+
+    assert len(results) == 2
+    assert results[0].level == "error"
+    assert results[0].lineno == 3
+    assert results[0].message == 'Unknown lint rule: "random"'
+    assert results[1].level == "error"
+    assert results[1].lineno == 4
+    assert results[1].message == 'Pattern "foo" not found in .gitignore'
+
+
+if __name__ == "__main__":
+    mozunit.main()