Backed out changeset fb19de542f01 (bug 1320607) for causing python failures complaining about test_devtools_database_parsing . CLOSED TREE

.eslintignore

@@ -78,6 +78,7 @@ devtools/client/debugger/webpack.config.js
 devtools/client/preferences/
 
 # Ignore devtools generated code
+devtools/shared/css/generated/properties-db.js
 devtools/client/webconsole/test/node/fixtures/stubs/*.js
 !devtools/client/webconsole/test/node/fixtures/stubs/index.js
 devtools/client/shared/components/test/node/stubs/reps/*.js

.prettierignore

@@ -1054,6 +1054,7 @@ devtools/client/debugger/webpack.config.js
 devtools/client/preferences/
 
 # Ignore devtools generated code
+devtools/shared/css/generated/properties-db.js
 devtools/client/webconsole/test/node/fixtures/stubs/*.js
 !devtools/client/webconsole/test/node/fixtures/stubs/index.js
 devtools/client/shared/components/test/node/stubs/reps/*.js

devtools/client/fronts/css-properties.js

@@ -18,6 +18,12 @@ loader.lazyRequireGetter(
   "resource://devtools/shared/css/color-db.js",
   true
 );
+loader.lazyRequireGetter(
+  this,
+  "CSS_PROPERTIES_DB",
+  "resource://devtools/shared/css/properties-db.js",
+  true
+);
 loader.lazyRequireGetter(
   this,
   "CSS_TYPES",
@@ -175,6 +181,16 @@ function isCssVariable(input) {
   return !!input.match(IS_VARIABLE_TOKEN);
 }
 
+/**
+ * Get a client-side CssProperties. This is useful for dependencies in tests, or parts
+ * of the codebase that don't particularly need to match every known CSS property on
+ * the target.
+ * @return {CssProperties}
+ */
+function getClientCssProperties() {
+  return new CssProperties(normalizeCssData(CSS_PROPERTIES_DB));
+}
+
 /**
  * Even if the target has the cssProperties actor, the returned data may not be in the
  * same shape or have all of the data we need. This normalizes the data and fills in
@@ -184,9 +200,49 @@ function isCssVariable(input) {
  */
 function normalizeCssData(db) {
   // If there is a `from` attributes, it means that it comes from RDP
-  // and it is not the client `generateCssProperties()` object passed by tests.
+  // and it is not the client CSS_PROPERTIES_DB object.
+  // (prevent comparing to CSS_PROPERTIES_DB to avoid loading client database)
   if (typeof db.from == "string") {
-    // This is where to put backward compat tweaks here to support old runtimes.
+    const missingSupports = !db.properties.color.supports;
+    const missingValues = !db.properties.color.values;
+    const missingSubproperties = !db.properties.background.subproperties;
+    const missingIsInherited = !db.properties.font.isInherited;
+
+    const missingSomething =
+      missingSupports ||
+      missingValues ||
+      missingSubproperties ||
+      missingIsInherited;
+
+    if (missingSomething) {
+      for (const name in db.properties) {
+        // Skip the current property if we can't find it in CSS_PROPERTIES_DB.
+        if (typeof CSS_PROPERTIES_DB.properties[name] !== "object") {
+          continue;
+        }
+
+        // Add "supports" information to the css properties if it's missing.
+        if (missingSupports) {
+          db.properties[name].supports =
+            CSS_PROPERTIES_DB.properties[name].supports;
+        }
+        // Add "values" information to the css properties if it's missing.
+        if (missingValues) {
+          db.properties[name].values =
+            CSS_PROPERTIES_DB.properties[name].values;
+        }
+        // Add "subproperties" information to the css properties if it's missing.
+        if (missingSubproperties) {
+          db.properties[name].subproperties =
+            CSS_PROPERTIES_DB.properties[name].subproperties;
+        }
+        // Add "isInherited" information to the css properties if it's missing.
+        if (missingIsInherited) {
+          db.properties[name].isInherited =
+            CSS_PROPERTIES_DB.properties[name].isInherited;
+        }
+      }
+    }
   }
 
   reattachCssColorValues(db);
@@ -215,8 +271,7 @@ function reattachCssColorValues(db) {
 
 module.exports = {
   CssPropertiesFront,
+  getClientCssProperties,
   isCssVariable,
-  CssProperties,
-  normalizeCssData,
 };
 registerFront(CssPropertiesFront);

devtools/client/inspector/shared/test/browser_styleinspector_output-parser.js

@@ -9,6 +9,9 @@
 // tested with an xpcshell test as the output-parser requires the DOM to work.
 
 const OutputParser = require("resource://devtools/client/shared/output-parser.js");
+const {
+  getClientCssProperties,
+} = require("resource://devtools/client/fronts/css-properties.js");
 
 const COLOR_CLASS = "color-class";
 const URL_CLASS = "url-class";

devtools/client/shared/sourceeditor/css-autocompleter.js

@@ -8,6 +8,9 @@ const {
   cssTokenizer,
   cssTokenizerWithLineColumn,
 } = require("resource://devtools/shared/css/parsing-utils.js");
+const {
+  getClientCssProperties,
+} = require("resource://devtools/client/fronts/css-properties.js");
 
 /**
  * Here is what this file (+ css-parsing-utils.js) do.
@@ -86,7 +89,8 @@ const SELECTOR_STATES = {
 function CSSCompleter(options = {}) {
   this.walker = options.walker;
   this.maxEntries = options.maxEntries || 15;
-  this.cssProperties = options.cssProperties;
+  // If no css properties database is passed in, default to the client list.
+  this.cssProperties = options.cssProperties || getClientCssProperties();
 
   this.propertyNames = this.cssProperties.getNames().sort();
 

devtools/client/shared/sourceeditor/test/browser_css_autocompletion.js

@@ -93,10 +93,9 @@ async function runTests() {
   const target = await createAndAttachTargetForTab(gBrowser.selectedTab);
   inspector = await target.getFront("inspector");
   const walker = inspector.walker;
-  const cssPropertiesFront = await target.getFront("cssProperties");
   completer = new CSSCompleter({
     walker,
-    cssProperties: cssPropertiesFront.cssProperties,
+    cssProperties: getClientCssProperties(),
   });
   await checkStateAndMoveOn();
   await completer.walker.release();

devtools/client/shared/sourceeditor/test/browser_editor_autocomplete_events.js

@@ -16,13 +16,12 @@ async function runTests() {
   const target = await createAndAttachTargetForTab(gBrowser.selectedTab);
   const inspector = await target.getFront("inspector");
   const walker = inspector.walker;
-  const cssPropertiesFront = await target.getFront("cssProperties");
   const { ed, win, edWin } = await setup({
     autocomplete: true,
     mode: Editor.modes.css,
     autocompleteOpts: {
       walker,
-      cssProperties: cssPropertiesFront.cssProperties,
+      cssProperties: getClientCssProperties(),
     },
   });
   await testMouse(ed, edWin);

devtools/client/shared/sourceeditor/test/head.js

@@ -13,6 +13,9 @@ Services.scriptloader.loadSubScript(
 );
 
 const Editor = require("resource://devtools/client/shared/sourceeditor/editor.js");
+const {
+  getClientCssProperties,
+} = require("resource://devtools/client/fronts/css-properties.js");
 
 function promiseWaitForFocus(el) {
   return new Promise(resolve => waitForFocus(resolve, el));

devtools/client/shared/test/browser_outputparser.js

@@ -3,6 +3,10 @@
 
 "use strict";
 
+const {
+  getClientCssProperties,
+} = require("resource://devtools/client/fronts/css-properties.js");
+
 add_task(async function () {
   await pushPref("layout.css.backdrop-filter.enabled", true);
   await pushPref("layout.css.individual-transform.enabled", true);

devtools/client/shared/test/shared-head.js

@@ -2207,24 +2207,3 @@ function logCssCompatDataPropertiesWithoutMDNUrl() {
   }
   walk(cssPropertiesCompatData);
 }
-
-/**
- * Craft a CssProperties instance without involving RDP for tests
- * manually spawning OutputParser, CssCompleter, Editor...
- *
- * Otherwise this should instead be fetched from CssPropertiesFront.
- *
- * @return {CssProperties}
- */
-function getClientCssProperties() {
-  const {
-    generateCssProperties,
-  } = require("resource://devtools/server/actors/css-properties.js");
-  const {
-    CssProperties,
-    normalizeCssData,
-  } = require("resource://devtools/client/fronts/css-properties.js");
-  return new CssProperties(
-    normalizeCssData({ properties: generateCssProperties() })
-  );
-}

devtools/shared/css/generated/generate-properties-db.js

@@ -0,0 +1,51 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/*
+ * This is an xpcshell script that runs to generate a static list of CSS properties
+ * as known by the platform. It is run from ./mach_commands.py by running
+ * `mach devtools-css-db`.
+ */
+var { require } = ChromeUtils.importESModule(
+  "resource://devtools/shared/loader/Loader.sys.mjs"
+);
+var {
+  generateCssProperties,
+} = require("resource://devtools/server/actors/css-properties.js");
+
+// xpcshell can output extra information, so place some delimiter text between
+// the output of the css properties database.
+dump("DEVTOOLS_CSS_DB_DELIMITER");
+
+// Output JSON
+dump(
+  JSON.stringify({
+    cssProperties: cssProperties(),
+  })
+);
+
+dump("DEVTOOLS_CSS_DB_DELIMITER");
+
+/*
+ * A list of CSS Properties and their various characteristics. This is used on the
+ * client-side when the CssPropertiesActor is not found, or when the client and server
+ * are the same version. A single property takes the form:
+ *
+ *  "animation": {
+ *    "isInherited": false,
+ *    "supports": [ 7, 9, 10 ]
+ *  }
+ */
+function cssProperties() {
+  const properties = generateCssProperties();
+  for (const key in properties) {
+    // Ignore OS-specific properties
+    if (key.includes("-moz-osx-")) {
+      properties[key] = undefined;
+    }
+  }
+  return properties;
+}

devtools/shared/css/generated/mach_commands.py

@@ -0,0 +1,119 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+This script implements the `mach devtools-css-db` command. It runs an xpcshell
+script that uses InspectorUtils to query the CSS properties used by the browser.
+This information is used to generate the properties-db.js file.
+"""
+
+import json
+import logging
+import os
+import runpy
+import sys
+import string
+import subprocess
+from mozbuild import shellutil
+from mozbuild.base import (
+    MozbuildObject,
+    BinaryNotFoundException,
+)
+from mach.decorators import (
+    Command,
+)
+
+
+def resolve_path(start, relativePath):
+    """Helper to resolve a path from a start, and a relative path"""
+    return os.path.normpath(os.path.join(start, relativePath))
+
+
+def stringify(obj):
+    """Helper to stringify to JSON"""
+    return json.dumps(obj, sort_keys=True, indent=2, separators=(",", ": "))
+
+
+@Command(
+    "devtools-css-db",
+    category="post-build",
+    description="Rebuild the devtool's static css properties database.",
+)
+def generate_css_db(command_context):
+    """Generate the static css properties database for devtools and write it to file."""
+
+    print("Re-generating the css properties database...")
+    db = get_properties_db_from_xpcshell(command_context)
+    if not db:
+        return 1
+
+    output_template(
+        command_context,
+        {
+            "cssProperties": stringify(db["cssProperties"]),
+        },
+    )
+
+
+def get_properties_db_from_xpcshell(command_context):
+    """Generate the static css properties db for devtools from an xpcshell script."""
+    build = MozbuildObject.from_environment()
+
+    # Get the paths
+    script_path = resolve_path(
+        command_context.topsrcdir,
+        "devtools/shared/css/generated/generate-properties-db.js",
+    )
+    gre_path = resolve_path(command_context.topobjdir, "dist/bin")
+    browser_path = resolve_path(command_context.topobjdir, "dist/bin/browser")
+    try:
+        xpcshell_path = build.get_binary_path(what="xpcshell")
+    except BinaryNotFoundException as e:
+        command_context.log(
+            logging.ERROR, "devtools-css-db", {"error": str(e)}, "ERROR: {error}"
+        )
+        command_context.log(
+            logging.INFO, "devtools-css-db", {"help": e.help()}, "{help}"
+        )
+        return None
+
+    print(browser_path)
+
+    sub_env = dict(os.environ)
+    if sys.platform.startswith("linux"):
+        sub_env["LD_LIBRARY_PATH"] = gre_path
+
+    # Run the xcpshell script, and set the appdir flag to the browser path so that
+    # we have the proper dependencies for requiring the loader.
+    contents = subprocess.check_output(
+        [xpcshell_path, "-g", gre_path, "-a", browser_path, script_path],
+        env=sub_env,
+    )
+    # Extract just the output between the delimiters as the xpcshell output can
+    # have extra output that we don't want.
+    contents = contents.decode().split("DEVTOOLS_CSS_DB_DELIMITER")[1]
+
+    return json.loads(contents)
+
+
+def output_template(command_context, substitutions):
+    """Output a the properties-db.js from a template."""
+    js_template_path = resolve_path(
+        command_context.topsrcdir,
+        "devtools/shared/css/generated/properties-db.js.in",
+    )
+    destination_path = resolve_path(
+        command_context.topsrcdir, "devtools/shared/css/generated/properties-db.js"
+    )
+
+    with open(js_template_path, "rb") as handle:
+        js_template = handle.read().decode()
+
+    preamble = "/* THIS IS AN AUTOGENERATED FILE.  DO NOT EDIT */\n\n"
+    contents = string.Template(js_template).substitute(substitutions)
+
+    with open(destination_path, "wb") as destination:
+        destination.write(preamble.encode() + contents.encode())
+
+    print("The database was successfully generated at " + destination_path)

devtools/shared/css/generated/moz.build

@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DevToolsModules(
+    "properties-db.js",
+)

devtools/shared/css/generated/properties-db.js.in

@@ -0,0 +1,16 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+/**
+ * This file is automatically generated by `mach devtools-css-db`. It contains
+ * a static list of CSS properties that can be computed by Gecko. The actual script
+ * to generate these files can be found at
+ * devtools/shared/css/generated/generate-properties-db.js.
+ */
+
+/**
+ * A list of CSS Properties and their various characteristics.
+ */
+exports.CSS_PROPERTIES = ${cssProperties};

devtools/shared/css/moz.build

@@ -4,10 +4,15 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+DIRS += [
+    "generated",
+]
+
 DevToolsModules(
     "color-db.js",
     "color.js",
     "constants.js",
     "lexer.js",
     "parsing-utils.js",
+    "properties-db.js",
 )

devtools/shared/css/properties-db.js

@@ -0,0 +1,44 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/**
+ * This file contains static lists of CSS properties and values. Some of the small lists
+ * are edited manually, while the larger ones are generated by a script. The comments
+ * above each list indicates how it should be updated.
+ */
+
+let db;
+
+// Allow this require to fail in case it's been deleted in the process of running
+// `mach devtools-css-db` to regenerate the database.
+try {
+  db = require("resource://devtools/shared/css/generated/properties-db.js");
+} catch (error) {
+  console.error(
+    `If this error is being displayed and "mach devtools-css-db" is not ` +
+      `being run, then it needs to be fixed.`,
+    error
+  );
+  db = {
+    CSS_PROPERTIES: {},
+  };
+}
+
+/**
+ * A list of CSS Properties and their various characteristics. This is used on the
+ * client-side when the CssPropertiesActor is not found, or when the client and server
+ * are the same version. A single property takes the form:
+ *
+ *  "animation": {
+ *    "isInherited": false,
+ *    "supports": [ 7, 9, 10 ]
+ *  }
+ */
+exports.CSS_PROPERTIES = db.CSS_PROPERTIES;
+
+exports.CSS_PROPERTIES_DB = {
+  properties: db.CSS_PROPERTIES,
+};

devtools/shared/tests/xpcshell/test_css-properties-db.js

@@ -0,0 +1,171 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Test that the devtool's client-side CSS properties database is in sync with the values
+ * on the platform (in Nightly only). If they are not, then `mach devtools-css-db` needs
+ * to be run to make everything up to date. Nightly, aurora, beta, and release may have
+ * different CSS properties and values. These are based on preferences and compiler flags.
+ *
+ * This test broke uplifts as the database needed to be regenerated every uplift. The
+ * combination of compiler flags and preferences means that it's too difficult to
+ * statically determine which properties are enabled between Firefox releases.
+ *
+ * Because of these difficulties, the database only needs to be up to date with Nightly.
+ * It is a fallback that is only used if the remote debugging protocol doesn't support
+ * providing a CSS database, so it's ok if the provided properties don't exactly match
+ * the inspected target in this particular case.
+ */
+
+"use strict";
+
+const {
+  CSS_PROPERTIES,
+} = require("resource://devtools/shared/css/generated/properties-db.js");
+const PREFERENCES = InspectorUtils.getCSSPropertyPrefs();
+const {
+  generateCssProperties,
+} = require("resource://devtools/server/actors/css-properties.js");
+const { Preferences } = ChromeUtils.importESModule(
+  "resource://gre/modules/Preferences.sys.mjs"
+);
+
+function run_test() {
+  const propertiesErrorMessage =
+    "If this assertion fails, then the client side CSS " +
+    "properties list in devtools is out of sync with the " +
+    "CSS properties on the platform. To fix this " +
+    "assertion run `mach devtools-css-db` to re-generate " +
+    "the client side properties.";
+
+  /**
+   * Check that the platform and client match for the details on their CSS properties.
+   * Enumerate each property to aid in debugging. Sometimes these properties don't
+   * completely agree due to differences in preferences. Check the currently set
+   * preference for that property to see if it's enabled.
+   */
+  const platformProperties = generateCssProperties();
+
+  for (const propertyName in CSS_PROPERTIES) {
+    const platformProperty = platformProperties[propertyName];
+    const clientProperty = CSS_PROPERTIES[propertyName];
+    const deepEqual = isJsonDeepEqual(platformProperty, clientProperty);
+
+    if (deepEqual) {
+      ok(true, `The static database and platform match for "${propertyName}".`);
+    } else {
+      ok(
+        false,
+        `The static database and platform do not match for ` +
+          `
+        "${propertyName}". ${propertiesErrorMessage}`
+      );
+    }
+  }
+
+  /**
+   * Check that the list of properties on the platform and client are the same. If
+   * they are not, check that there may be preferences that are disabling them on the
+   * target platform.
+   */
+  const mismatches = getKeyMismatches(platformProperties, CSS_PROPERTIES)
+    // Filter out OS-specific properties.
+    .filter(name => name && !name.includes("-moz-osx-"));
+
+  if (mismatches.length === 0) {
+    ok(
+      true,
+      "No client and platform CSS property database mismatches were found."
+    );
+  }
+
+  mismatches.forEach(propertyName => {
+    if (getPreference(propertyName) === false) {
+      ok(
+        true,
+        `The static database and platform do not agree on the property ` +
+          `"${propertyName}" This is ok because it is currently disabled through ` +
+          `a preference.`
+      );
+    } else {
+      ok(
+        false,
+        `The static database and platform do not agree on the property ` +
+          `"${propertyName}" ${propertiesErrorMessage}`
+      );
+    }
+  });
+}
+
+/**
+ * Check JSON-serializable objects for deep equality.
+ */
+function isJsonDeepEqual(a, b) {
+  // Handle primitives.
+  if (a === b) {
+    return true;
+  }
+
+  // Handle arrays.
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) {
+      return false;
+    }
+    for (let i = 0; i < a.length; i++) {
+      if (!isJsonDeepEqual(a[i], b[i])) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  // Handle objects
+  if (typeof a === "object" && typeof b === "object") {
+    for (const key in a) {
+      if (!isJsonDeepEqual(a[key], b[key])) {
+        return false;
+      }
+    }
+
+    return Object.keys(a).length === Object.keys(b).length;
+  }
+
+  // Not something handled by these cases, therefore not equal.
+  return false;
+}
+
+/**
+ * Take the keys of two objects, and return the ones that don't match.
+ *
+ * @param {Object} a
+ * @param {Object} b
+ * @return {Array} keys
+ */
+function getKeyMismatches(a, b) {
+  const aNames = Object.keys(a);
+  const bNames = Object.keys(b);
+  const aMismatches = aNames.filter(key => !bNames.includes(key));
+  const bMismatches = bNames.filter(key => {
+    return !aNames.includes(key) && !aMismatches.includes(key);
+  });
+
+  return aMismatches.concat(bMismatches);
+}
+
+/**
+ * Get the preference value of whether this property is enabled. Returns an empty string
+ * if no preference exists.
+ *
+ * @param {String} propertyName
+ * @return {Boolean|undefined}
+ */
+function getPreference(propertyName) {
+  const preference = PREFERENCES.find(({ name, pref }) => {
+    return name === propertyName && !!pref;
+  });
+
+  if (preference) {
+    return Preferences.get(preference.pref);
+  }
+  return undefined;
+}

devtools/shared/tests/xpcshell/xpcshell.ini

@@ -11,6 +11,12 @@ support-files =
 [test_assert.js]
 [test_debugger_client.js]
 [test_csslexer.js]
+[test_css-properties-db.js]
+# This test only enforces that the CSS database is up to date with nightly. The DB is
+# only used when inspecting a target that doesn't support the getCSSDatabase actor.
+# CSS properties are behind compile-time flags, and there is no automatic rebuild
+# process for uplifts, so this test breaks on uplift.
+run-if = nightly_build
 [test_eventemitter_abort_controller.js]
 [test_eventemitter_basic.js]
 [test_eventemitter_destroy.js]

python/mach/mach/command_util.py

@@ -83,6 +83,9 @@ MACH_COMMANDS = {
     "data-review": MachCommandReference(
         "toolkit/components/glean/build_scripts/mach_commands.py"
     ),
+    "devtools-css-db": MachCommandReference(
+        "devtools/shared/css/generated/mach_commands.py"
+    ),
     "doc": MachCommandReference("tools/moztreedocs/mach_commands.py"),
     "doctor": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),
     "environment": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"),