Bug 1804178 - [6/6] Switch from pytoml to toml in (almost) all other contexts r=firefox-build-system-reviewers,releng-reviewers,glandium,ahal

Convert all other imports of `pytoml` within mozilla-central to imports
of `toml`.

As both libraries patterned their API on that of the Python standard-
library module `json`, this is mostly a straight replace, with two
caveats:

  * pytoml's exception messages when loading a file provide the file
    name; toml's do not. Some additional machinery has been added or
    repurposed in a few places to ensure that the relevant filename is
    printed.

  * In `python_lockfile.py`, the order of arguments to `dump` needed to
    be reversed. (`toml` follows `json` in this; `pytoml` differs.)

This patchset does not remove pytoml from `requirements.in`, as `pytoml`
is still used by `compare-locales`.

Differential Revision: https://phabricator.services.mozilla.com/D164155

build/RunCbindgen.py

@@ -9,15 +9,15 @@ import subprocess
 
 import buildconfig
 import mozpack.path as mozpath
-import pytoml
 import six
+import toml
 
 
 # Try to read the package name or otherwise assume same name as the crate path.
 def _get_crate_name(crate_path):
     try:
         with open(mozpath.join(crate_path, "Cargo.toml"), encoding="utf-8") as f:
-            return pytoml.load(f)["package"]["name"]
+            return toml.load(f)["package"]["name"]
     except Exception:
         return mozpath.basename(crate_path)
 

python/l10n/mozxchannel/source.py

@@ -4,7 +4,7 @@
 
 from compare_locales import paths, mozpath
 from compare_locales.paths.matcher import expand
-import pytoml as toml
+import toml
 
 from .projectconfig import generate_filename
 
@@ -81,4 +81,8 @@ class HgTOMLParser(paths.TOMLParser):
             data = self.repo.cat(files=[local_path.encode("utf-8")], rev=self.rev)
         except Exception:
             raise paths.ConfigNotFound(parse_ctx.path)
-        parse_ctx.data = toml.loads(data, filename=parse_ctx.path)
+
+        try:
+            parse_ctx.data = toml.loads(data)
+        except toml.TomlDecodeError as e:
+            raise RuntimeError(f"In file '{parse_ctx.path}':\n  {e!s}") from e

python/mach/mach/python_lockfile.py

@@ -6,7 +6,7 @@ import subprocess
 import sys
 from pathlib import Path
 
-import pytoml
+import toml
 from packaging.requirements import Requirement
 
 
@@ -63,7 +63,7 @@ class PoetryHandle:
 
         pyproject = {"tool": {"poetry": poetry_config}}
         with open(self._work_dir / "pyproject.toml", "w") as pyproject_file:
-            pytoml.dump(pyproject_file, pyproject)
+            toml.dump(pyproject, pyproject_file)
 
         self._run_poetry(["lock"] + (["--no-update"] if not do_update else []))
         self._run_poetry(["export", "-o", "requirements.txt"])

python/mozbuild/mozbuild/frontend/emitter.py

@@ -17,7 +17,7 @@ from mozbuild.util import memoize, OrderedDefaultDict
 
 import mozpack.path as mozpath
 import mozinfo
-import pytoml
+import toml
 
 from .data import (
     BaseRustProgram,
@@ -523,7 +523,7 @@ class TreeMetadataEmitter(LoggingMixin):
                 "No Cargo.toml file found in %s" % cargo_file, context
             )
         with open(cargo_file, "r") as f:
-            return pytoml.load(f), cargo_file
+            return toml.load(f), cargo_file
 
     def _verify_deps(
         self, context, crate_dir, crate_name, dependencies, description="Dependency"

taskcluster/scripts/misc/repack_rust.py

@@ -22,8 +22,8 @@ import tarfile
 import textwrap
 from contextlib import contextmanager
 
-import pytoml as toml
 import requests
+import toml
 import zstandard
 
 
@@ -345,7 +345,7 @@ def fetch_manifest(channel="stable", host=None, targets=()):
     url = "https://static.rust-lang.org/dist%s/channel-rust-%s.toml" % (prefix, channel)
     req = requests.get(url)
     req.raise_for_status()
-    manifest = toml.loads(req.content)
+    manifest = toml.loads(req.text)
     if manifest["manifest-version"] != "2":
         raise NotImplementedError(
             "Unrecognized manifest version %s." % manifest["manifest-version"]

toolkit/components/featuregates/gen_feature_definitions.py

@@ -4,9 +4,9 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
-import pytoml
 import re
 import sys
+import toml
 import voluptuous
 import voluptuous.humanize
 from voluptuous import Schema, Optional, Any, All, Required, Length, Range, Msg, Match
@@ -77,7 +77,7 @@ class FeatureGateException(Exception):
         if self.filename is None:
             rv.append("unknown file:")
         else:
-            rv.append('file "{}":'.format(self.filename))
+            rv.append('file "{}":\n'.format(self.filename))
         rv.append(message)
         return " ".join(rv)
 
@@ -98,7 +98,7 @@ def process_files(filenames):
     for filename in filenames:
         try:
             with open(filename, "r") as f:
-                feature_data = pytoml.load(f)
+                feature_data = toml.load(f)
 
             voluptuous.humanize.validate_with_humanized_errors(
                 feature_data, feature_schema
@@ -107,12 +107,14 @@ def process_files(filenames):
             for feature_id, feature in feature_data.items():
                 feature["id"] = feature_id
                 features[feature_id] = expand_feature(feature)
-        except (voluptuous.error.Error, IOError, FeatureGateException) as e:
+        except (
+            voluptuous.error.Error,
+            IOError,
+            FeatureGateException,
+            toml.TomlDecodeError,
+        ) as e:
             # Wrap errors in enough information to know which file they came from
             errors.append(FeatureGateException(e, filename))
-        except pytoml.TomlError as e:
-            # Toml errors have file information already
-            errors.append(e)
 
     if errors:
         raise ExceptionGroup(errors)

toolkit/components/featuregates/test/python/test_gen_feature_definitions.py

@@ -5,7 +5,7 @@ from os import path
 from textwrap import dedent
 
 import mozunit
-import pytoml
+import toml
 import voluptuous
 from io import StringIO
 
@@ -83,7 +83,7 @@ class TestFeatureGateException(unittest.TestCase):
 
     def test_str_with_file(self):
         error = FeatureGateException("oops", filename="some/bad/file.txt")
-        assert str(error) == 'In file "some/bad/file.txt": oops'
+        assert str(error) == 'In file "some/bad/file.txt":\n oops'
 
     def test_repr_no_file(self):
         error = FeatureGateException("oops")
@@ -131,7 +131,7 @@ class TestProcessFiles(unittest.TestCase):
             process_files([filename])
         error_group = context.exception
         assert len(error_group.errors) == 1
-        assert type(error_group.errors[0]) == pytoml.TomlError
+        assert type(error_group.errors[0]) == FeatureGateException
 
     def test_empty_feature(self):
         filename = make_test_file_path("empty_feature")