Bug 1807831 - Add a check to make sure that we don't regress the number of warnings r=sylvestre,ahal DONTBUILD

Differential Revision: https://phabricator.services.mozilla.com/D167528

docs/config.yml

@@ -110,3 +110,4 @@ redirects:
 
 fatal warnings:
     - "WARNING: '([^']*)' reference target not found:((?!.rst).)*$"
+max_num_warnings: 6717

tools/moztreedocs/mach_commands.py

@@ -97,6 +97,11 @@ BASE_LINK = "http://gecko-docs.mozilla.org-l1.s3-website.us-west-2.amazonaws.com
     action="store_true",
     help="Enable fatal warnings.",
 )
+@CommandArgument(
+    "--check-num-warnings",
+    action="store_true",
+    help="Check that the upper bound on the number of warnings is respected.",
+)
 @CommandArgument("--verbose", action="store_true", help="Run Sphinx in verbose mode")
 def build_docs(
     command_context,
@@ -113,6 +118,7 @@ def build_docs(
     linkcheck=None,
     dump_trees=None,
     enable_fatal_warnings=False,
+    check_num_warnings=False,
     verbose=None,
 ):
     # TODO: Bug 1704891 - move the ESLint setup tools to a shared place.
@@ -169,14 +175,18 @@ def build_docs(
         )
     else:
         print("\nGenerated documentation:\n%s" % savedir)
+    msg = ""
 
     if enable_fatal_warnings:
-        fatal_warnings = _check_sphinx_warnings(warnings)
+        fatal_warnings = _check_sphinx_fatal_warnings(warnings)
         if fatal_warnings:
-            return die(
+            msg += f"Error: Got fatal warnings:\n{''.join(fatal_warnings)}"
-                "failed to generate documentation:\n "
+    if check_num_warnings:
-                f"Error: Got fatal warnings:\n{''.join(fatal_warnings)}"
+        num_new = _check_sphinx_num_warnings(warnings)
-            )
+        if num_new:
+            msg += f"Error: {num_new} new warnings"
+    if msg:
+        return die(f"failed to generate documentation:\n {msg}")
 
     # Upload the artifact containing the link to S3
     # This would be used by code-review to post the link to Phabricator
@@ -297,7 +307,7 @@ def _run_sphinx(docdir, savedir, config=None, fmt="html", jobs=None, verbose=Non
             print(ex)
 
 
-def _check_sphinx_warnings(warnings):
+def _check_sphinx_fatal_warnings(warnings):
     with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh:
         fatal_warnings_src = yaml.safe_load(fh)["fatal warnings"]
     fatal_warnings_regex = [re.compile(item) for item in fatal_warnings_src]
@@ -308,6 +318,16 @@ def _check_sphinx_warnings(warnings):
     return fatal_warnings
 
 
+def _check_sphinx_num_warnings(warnings):
+    # warnings file contains other strings as well
+    num_warnings = len([w for w in warnings if "WARNING" in w])
+    with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh:
+        max_num = yaml.safe_load(fh)["max_num_warnings"]
+    if num_warnings > max_num:
+        return num_warnings - max_num
+    return None
+
+
 def manager():
     from moztreedocs import manager