Bug 1872918 - Collect .d.json typescript info from xpidl r=mossop,nika

Differential Revision: https://phabricator.services.mozilla.com/D197618

.prettierignore

@@ -1490,3 +1490,4 @@ toolkit/components/uniffi-bindgen-gecko-js/fixtures/generated
 tools/browsertime/package.json
 tools/browsertime/package-lock.json
 try_task_config.json
+xpcom/idl-parser/xpidl/fixtures/xpctest.d.json

python/mozbuild/mozbuild/action/xpidl-process.py

@@ -14,7 +14,7 @@ import sys
 import six
 from buildconfig import topsrcdir
 from mozpack import path as mozpath
-from xpidl import jsonxpt
+from xpidl import jsonxpt, typescript
 from xpidl.header import print_header
 from xpidl.rust import print_rust_bindings
 from xpidl.rust_macros import print_rust_macros_bindings
@@ -39,6 +39,8 @@ def process(
     p = IDLParser()
 
     xpts = []
+    ts_data = []
+
     mk = Makefile()
     rule = mk.create_rule()
 
@@ -63,6 +65,7 @@ def process(
         rs_bt_path = os.path.join(xpcrs_dir, "bt", "%s.rs" % stem)
 
         xpts.append(jsonxpt.build_typelib(idl))
+        ts_data.append(typescript.ts_source(idl))
 
         rule.add_dependencies(six.ensure_text(s) for s in idl.deps)
 
@@ -94,6 +97,13 @@ def process(
     with open(xpt_path, "w", encoding="utf-8", newline="\n") as fh:
         jsonxpt.write(jsonxpt.link(xpts), fh)
 
+    # NOTE: Make doesn't know about .d.json files, but we can piggy-back
+    # on XPT generation for now, as conceptually they contain the same
+    # information, and should be built together in all cases.
+    ts_path = os.path.join(xpt_dir, f"{module}.d.json")
+    with open(ts_path, "w", encoding="utf-8", newline="\n") as fh:
+        typescript.write(ts_data, fh)
+
     rule.add_targets([six.ensure_text(xpt_path)])
     if deps_dir:
         deps_path = os.path.join(deps_dir, "%s.pp" % module)

tools/rewriting/Generated.txt

@@ -37,3 +37,4 @@ toolkit/components/uniffi-bindgen-gecko-js/fixtures/generated
 tools/browsertime/package.json
 tools/browsertime/package-lock.json
 try_task_config.json
+xpcom/idl-parser/xpidl/fixtures/xpctest.d.json

xpcom/base/nsrootidl.idl

@@ -35,18 +35,21 @@ class Promise;
 #if 0
 %}
 
-typedef boolean             bool   ;
+// [substitute] typedefs emit the underlying builtin type directly, and
-typedef octet               uint8_t  ;
+// avoid polluting bindings for other languages with C++ stdint types.
-typedef unsigned short      uint16_t ;
-typedef unsigned short      char16_t;
-typedef unsigned long       uint32_t ;
-typedef unsigned long long  uint64_t ;
-typedef long long           PRTime   ;
-typedef short               int16_t  ;
-typedef long                int32_t  ;
-typedef long long           int64_t  ;
 
-typedef unsigned long       nsresult ;
+[substitute]  typedef boolean             bool     ;
+[substitute]  typedef octet               uint8_t  ;
+[substitute]  typedef unsigned short      uint16_t ;
+[substitute]  typedef unsigned long       uint32_t ;
+[substitute]  typedef unsigned long long  uint64_t ;
+[substitute]  typedef short               int16_t  ;
+[substitute]  typedef long                int32_t  ;
+[substitute]  typedef long long           int64_t  ;
+
+              typedef unsigned short      char16_t ;
+              typedef unsigned long       nsresult ;
+              typedef long long           PRTime   ;
 
 // If we ever want to use `size_t` in scriptable interfaces, this will need to
 // be built into the xpidl compiler, as the size varies based on platform.

xpcom/idl-parser/xpidl/header.py

@@ -339,6 +339,8 @@ def print_header(idl, fd, filename, relpath):
             write_interface(p, fd)
             continue
         if p.kind == "typedef":
+            if p.substitute:
+                continue
             printComments(fd, p.doccomments, "")
             fd.write("typedef %s %s;\n\n" % (p.realtype.nativeType("in"), p.name))
 

xpcom/idl-parser/xpidl/runtests.py

@@ -5,6 +5,8 @@
 #
 # Unit tests for xpidl.py
 
+import json
+import os
 import sys
 
 # Hack: the first entry in sys.path is the directory containing the script.
@@ -16,7 +18,7 @@ import unittest
 
 import mozunit
 
-from xpidl import header, xpidl
+from xpidl import header, typescript, xpidl
 
 
 class TestParser(unittest.TestCase):
@@ -253,5 +255,42 @@ attribute long bar;
             self.assertEqual(e.args[0], ("cannot find symbol 'Y'"))
 
 
+class TestTypescript(unittest.TestCase):
+    """A few basic smoke tests for typescript generation."""
+
+    dir = os.path.dirname(__file__)
+    src = os.path.join(dir, "..", "..", "..")
+
+    # We use the xpctest.xpt *.idl files from:
+    tests_dir = os.path.join(src, "js/xpconnect/tests/idl")
+    files = [
+        "xpctest_attributes.idl",
+        "xpctest_bug809674.idl",
+        "xpctest_cenums.idl",
+        "xpctest_interfaces.idl",
+        "xpctest_params.idl",
+        "xpctest_returncode.idl",
+        "xpctest_utils.idl",
+    ]
+
+    fixtures = os.path.join(dir, "fixtures")
+    inc_dirs = [os.path.join(src, "xpcom/base")]
+
+    def setUp(self):
+        self.parser = xpidl.IDLParser()
+
+    def test_d_json(self):
+        mods = []
+        for file in self.files:
+            path = os.path.join(self.tests_dir, file)
+            idl = self.parser.parse(open(path).read(), path)
+            idl.resolve(self.inc_dirs, self.parser, {})
+            mods.append(typescript.ts_source(idl))
+
+        result = json.dumps(mods, indent=2, sort_keys=True)
+        expected = open(os.path.join(self.fixtures, "xpctest.d.json")).read()
+        self.assertEqual(result, expected, "types data json does not match")
+
+
 if __name__ == "__main__":
     mozunit.main(runwith="unittest")

xpcom/idl-parser/xpidl/rust.py

@@ -363,8 +363,8 @@ def print_rust_bindings(idl, fd, relpath):
 
         if p.kind == "typedef":
             try:
-                # We have to skip the typedef of bool to bool (it doesn't make any sense anyways)
+                # Skip bool and C++ stdint typedefs marked with [substitute].
-                if p.name == "bool":
+                if p.substitute:
                     continue
 
                 if printdoccomments:

xpcom/idl-parser/xpidl/typescript.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# typescript.py - Collect .d.json TypeScript info from xpidl.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+
+import mozpack.path as mozpath
+
+from xpidl import xpidl
+
+
+def ts_enum(e):
+    variants = [{"name": v.name, "value": v.getValue()} for v in e.variants]
+    return {"id": e.basename, "variants": variants}
+
+
+def ts_attribute(a):
+    return {"name": a.name, "type": a.realtype.tsType(), "readonly": a.readonly}
+
+
+def ts_method(m):
+    args = []
+    for p in m.params:
+        if p.iid_is and not p.retval:
+            raise xpidl.TSNoncompat(f"{m.name} has unsupported iid_is argument")
+        args.append({"name": p.name, "optional": p.optional, "type": p.tsType()})
+
+    iid_is = None
+    type = m.realtype.tsType()
+    if args and m.params[-1].retval:
+        type = args.pop()["type"]
+        iid_is = m.params[-1].iid_is
+
+    return {"name": m.name, "type": type, "iid_is": iid_is, "args": args}
+
+
+def ts_interface(iface):
+    enums = []
+    consts = []
+    members = []
+
+    for m in iface.members:
+        try:
+            if isinstance(m, xpidl.CEnum):
+                enums.append(ts_enum(m))
+            elif isinstance(m, xpidl.ConstMember):
+                consts.append({"name": m.name, "value": m.getValue()})
+            elif isinstance(m, xpidl.Attribute):
+                members.append(ts_attribute(m))
+            elif isinstance(m, xpidl.Method):
+                members.append(ts_method(m))
+        except xpidl.TSNoncompat:
+            # Omit member if any type is unsupported.
+            pass
+
+    return {
+        "id": iface.name,
+        "base": iface.base,
+        "callable": iface.attributes.function,
+        "enums": enums,
+        "consts": consts,
+        "members": members,
+    }
+
+
+def ts_typedefs(idl):
+    for p in idl.getNames():
+        if isinstance(p, xpidl.Typedef) and not p.substitute:
+            try:
+                yield (p.name, p.realtype.tsType())
+            except xpidl.TSNoncompat:
+                pass
+
+
+def ts_source(idl):
+    """Collect typescript interface .d.json from a source idl file."""
+    root = mozpath.join(mozpath.dirname(__file__), "../../..")
+    return {
+        "path": mozpath.relpath(idl.productions[0].location._file, root),
+        "interfaces": [
+            ts_interface(p)
+            for p in idl.productions
+            if isinstance(p, xpidl.Interface) and p.attributes.scriptable
+        ],
+        "typedefs": sorted(ts_typedefs(idl)),
+    }
+
+
+def write(d_json, fd):
+    """Write json type info into fd"""
+    json.dump(d_json, fd, indent=2, sort_keys=True)

xpcom/idl-parser/xpidl/xpidl.py

@@ -122,10 +122,13 @@ class Builtin(object):
     kind = "builtin"
     location = BuiltinLocation
 
-    def __init__(self, name, nativename, rustname, signed=False, maybeConst=False):
+    def __init__(
+        self, name, nativename, rustname, tsname, signed=False, maybeConst=False
+    ):
         self.name = name
         self.nativename = nativename
         self.rustname = rustname
+        self.tsname = tsname
         self.signed = signed
         self.maybeConst = maybeConst
 
@@ -171,28 +174,37 @@ class Builtin(object):
 
         return "%s%s" % ("*mut " if "out" in calltype else "", rustname)
 
+    def tsType(self):
+        if self.tsname:
+            return self.tsname
+
+        raise TSNoncompat(f"Builtin type {self.name} unsupported in TypeScript")
+
 
 builtinNames = [
-    Builtin("boolean", "bool", "bool"),
+    Builtin("boolean", "bool", "bool", "boolean"),
-    Builtin("void", "void", "libc::c_void"),
+    Builtin("void", "void", "libc::c_void", "void"),
-    Builtin("octet", "uint8_t", "u8", False, True),
+    Builtin("octet", "uint8_t", "u8", "u8", False, True),
-    Builtin("short", "int16_t", "i16", True, True),
+    Builtin("short", "int16_t", "i16", "i16", True, True),
-    Builtin("long", "int32_t", "i32", True, True),
+    Builtin("long", "int32_t", "i32", "i32", True, True),
-    Builtin("long long", "int64_t", "i64", True, True),
+    Builtin("long long", "int64_t", "i64", "i64", True, True),
-    Builtin("unsigned short", "uint16_t", "u16", False, True),
+    Builtin("unsigned short", "uint16_t", "u16", "u16", False, True),
-    Builtin("unsigned long", "uint32_t", "u32", False, True),
+    Builtin("unsigned long", "uint32_t", "u32", "u32", False, True),
-    Builtin("unsigned long long", "uint64_t", "u64", False, True),
+    Builtin("unsigned long long", "uint64_t", "u64", "u64", False, True),
-    Builtin("float", "float", "libc::c_float", True, False),
+    Builtin("float", "float", "libc::c_float", "float"),
-    Builtin("double", "double", "libc::c_double", True, False),
+    Builtin("double", "double", "libc::c_double", "double"),
-    Builtin("char", "char", "libc::c_char", True, False),
+    Builtin("char", "char", "libc::c_char", "string"),
-    Builtin("string", "char *", "*const libc::c_char", False, False),
+    Builtin("string", "char *", "*const libc::c_char", "string"),
-    Builtin("wchar", "char16_t", "u16", False, False),
+    Builtin("wchar", "char16_t", "u16", "string"),
-    Builtin("wstring", "char16_t *", "*const u16", False, False),
+    Builtin("wstring", "char16_t *", "*const u16", "string"),
     # As seen in mfbt/RefCountType.h, this type has special handling to
     # maintain binary compatibility with MSCOM's IUnknown that cannot be
     # expressed in XPIDL.
     Builtin(
-        "MozExternalRefCountType", "MozExternalRefCountType", "MozExternalRefCountType"
+        "MozExternalRefCountType",
+        "MozExternalRefCountType",
+        "MozExternalRefCountType",
+        None,
     ),
 ]
 
@@ -308,6 +320,16 @@ class RustNoncompat(Exception):
         return self.reason
 
 
+class TSNoncompat(Exception):
+    """Raised when a type cannot be exposed to TypeScript."""
+
+    def __init__(self, reason):
+        self.reason = reason
+
+    def __str__(self):
+        return self.reason
+
+
 class IDLError(Exception):
     def __init__(self, message, location, warning=False, notes=None):
         self.message = message
@@ -458,12 +480,20 @@ class CDATA(object):
 class Typedef(object):
     kind = "typedef"
 
-    def __init__(self, type, name, location, doccomments):
+    def __init__(self, type, name, attlist, location, doccomments):
         self.type = type
         self.name = name
         self.location = location
         self.doccomments = doccomments
 
+        # C++ stdint types and the bool typedef from nsrootidl.idl are marked
+        # with [substitute], and emit as the underlying builtin type directly.
+        self.substitute = False
+        for name, value, aloc in attlist:
+            if name != "substitute" or value is not None:
+                raise IDLError(f"Unexpected attribute {name}({value})", aloc)
+            self.substitute = True
+
     def __eq__(self, other):
         return self.name == other.name and self.type == other.type
 
@@ -475,14 +505,26 @@ class Typedef(object):
             raise IDLError("Unsupported typedef target type", self.location)
 
     def nativeType(self, calltype):
+        if self.substitute:
+            return self.realtype.nativeType(calltype)
+
         return "%s %s" % (self.name, "*" if "out" in calltype else "")
 
     def rustType(self, calltype):
+        if self.substitute:
+            return self.realtype.rustType(calltype)
+
         if self.name == "nsresult":
             return "%s::nserror::nsresult" % ("*mut " if "out" in calltype else "")
 
         return "%s%s" % ("*mut " if "out" in calltype else "", self.name)
 
+    def tsType(self):
+        if self.substitute:
+            return self.realtype.tsType()
+
+        return self.name
+
     def __str__(self):
         return "typedef %s %s\n" % (self.type, self.name)
 
@@ -524,6 +566,9 @@ class Forward(object):
             return "Option<RefPtr<%s>>" % self.name
         return "%s*const %s" % ("*mut" if "out" in calltype else "", self.name)
 
+    def tsType(self):
+        return self.name
+
     def __str__(self):
         return "forward-declared %s\n" % self.name
 
@@ -701,6 +746,21 @@ class Native(object):
 
         raise RustNoncompat("native type %s unsupported" % self.nativename)
 
+    ts_special = {
+        "astring": "string",
+        "cstring": "string",
+        "jsval": "any",
+        "nsid": "nsID",
+        "promise": "Promise<any>",
+        "utf8string": "string",
+    }
+
+    def tsType(self):
+        if type := self.ts_special.get(self.specialtype, None):
+            return type
+
+        raise TSNoncompat(f"Native type {self.name} unsupported in TypeScript")
+
     def __str__(self):
         return "native %s(%s)\n" % (self.name, self.nativename)
 
@@ -749,6 +809,9 @@ class WebIDL(object):
         # Just expose the type as a void* - we can't do any better.
         return "%s*const libc::c_void" % ("*mut " if "out" in calltype else "")
 
+    def tsType(self):
+        return self.name
+
     def __str__(self):
         return "webidl %s\n" % self.name
 
@@ -923,6 +986,9 @@ class Interface(object):
             total += realbase.countEntries()
         return total
 
+    def tsType(self):
+        return self.name
+
 
 class InterfaceAttributes(object):
     uuid = None
@@ -1110,6 +1176,9 @@ class CEnum(object):
     def rustType(self, calltype):
         return "%s u%d" % ("*mut" if "out" in calltype else "", self.width)
 
+    def tsType(self):
+        return f"{self.iface.name}.{self.basename}"
+
     def __str__(self):
         body = ", ".join("%s = %s" % v for v in self.variants)
         return "\tcenum %s : %d { %s };\n" % (self.name, self.width, body)
@@ -1523,8 +1592,22 @@ class Param(object):
             self.name,
         )
 
+    def tsType(self):
+        # A generic retval param type needs special handling.
+        if self.retval and self.iid_is:
+            return "nsQIResult"
+
+        type = self.realtype.tsType()
+        if self.paramtype == "inout":
+            return f"InOutParam<{type}>"
+        if self.paramtype == "out":
+            return f"OutParam<{type}>"
+        return type
+
 
 class LegacyArray(object):
+    kind = "legacyarray"
+
     def __init__(self, basetype):
         self.type = basetype
         self.location = self.type.location
@@ -1555,6 +1638,9 @@ class LegacyArray(object):
             self.type.rustType("legacyelement"),
         )
 
+    def tsType(self):
+        return self.type.tsType() + "[]"
+
 
 class Array(object):
     kind = "array"
@@ -1594,6 +1680,9 @@ class Array(object):
         else:
             return base
 
+    def tsType(self):
+        return self.type.tsType() + "[]"
+
 
 TypeId = namedtuple("TypeId", "name params")
 
@@ -1751,12 +1840,13 @@ class IDLParser(object):
         p[0].insert(0, p[1])
 
     def p_typedef(self, p):
-        """typedef : TYPEDEF type IDENTIFIER ';'"""
+        """typedef : attributes TYPEDEF type IDENTIFIER ';'"""
         p[0] = Typedef(
-            type=p[2],
+            type=p[3],
-            name=p[3],
+            name=p[4],
-            location=self.getLocation(p, 1),
+            attlist=p[1]["attlist"],
-            doccomments=p.slice[1].doccomments,
+            location=self.getLocation(p, 2),
+            doccomments=getattr(p[1], "doccomments", []) + p.slice[2].doccomments,
         )
 
     def p_native(self, p):