forked from mirrors/gecko-dev
02a7b4ebdf
Allow-list all Python code in tree for use with the black linter, and re-format all code in-tree accordingly. To produce this patch I did all of the following: 1. Make changes to tools/lint/black.yml to remove include: stanza and update list of source extensions. 2. Run ./mach lint --linter black --fix 3. Make some ad-hoc manual updates to python/mozbuild/mozbuild/test/configure/test_configure.py -- it has some hard-coded line numbers that the reformat breaks. 4. Make some ad-hoc manual updates to `testing/marionette/client/setup.py`, `testing/marionette/harness/setup.py`, and `testing/firefox-ui/harness/setup.py`, which have hard-coded regexes that break after the reformat. 5. Add a set of exclusions to black.yml. These will be deleted in a follow-up bug (1672023). # ignore-this-changeset Differential Revision: https://phabricator.services.mozilla.com/D94045
298 lines
10 KiB
Python
298 lines
10 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
# Write out histogram information for C++. The histograms are defined
|
|
# in a file provided as a command-line argument.
|
|
|
|
from __future__ import print_function
|
|
from mozparsers.shared_telemetry_utils import static_assert, ParserError
|
|
from mozparsers import parse_histograms
|
|
|
|
import sys
|
|
import buildconfig
|
|
from os import path
|
|
|
|
COMPONENTS_PATH = path.abspath(
|
|
path.join(path.dirname(__file__), path.pardir, path.pardir)
|
|
)
|
|
sys.path.append(
|
|
path.join(COMPONENTS_PATH, "glean", "build_scripts", "glean_parser_ext")
|
|
)
|
|
from string_table import StringTable
|
|
|
|
banner = """/* This file is auto-generated, see gen_histogram_data.py. */
|
|
"""
|
|
|
|
|
|
def print_array_entry(
|
|
output,
|
|
histogram,
|
|
name_index,
|
|
exp_index,
|
|
label_index,
|
|
label_count,
|
|
key_index,
|
|
key_count,
|
|
store_index,
|
|
store_count,
|
|
):
|
|
if histogram.record_on_os(buildconfig.substs["OS_TARGET"]):
|
|
print(
|
|
" { %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %s, %s, %s, %s, %s, %s },"
|
|
% (
|
|
histogram.low(),
|
|
histogram.high(),
|
|
histogram.n_buckets(),
|
|
name_index,
|
|
exp_index,
|
|
label_count,
|
|
key_count,
|
|
store_count,
|
|
label_index,
|
|
key_index,
|
|
store_index,
|
|
" | ".join(histogram.record_in_processes_enum()),
|
|
"true" if histogram.keyed() else "false",
|
|
histogram.nsITelemetry_kind(),
|
|
histogram.dataset(),
|
|
" | ".join(histogram.products_enum()),
|
|
),
|
|
file=output,
|
|
)
|
|
|
|
|
|
def write_histogram_table(output, histograms):
|
|
string_table = StringTable()
|
|
|
|
label_table = []
|
|
label_count = 0
|
|
keys_table = []
|
|
keys_count = 0
|
|
store_table = []
|
|
total_store_count = 0
|
|
|
|
print("constexpr HistogramInfo gHistogramInfos[] = {", file=output)
|
|
for histogram in histograms:
|
|
name_index = string_table.stringIndex(histogram.name())
|
|
exp_index = string_table.stringIndex(histogram.expiration())
|
|
|
|
labels = histogram.labels()
|
|
label_index = 0
|
|
if len(labels) > 0:
|
|
label_index = label_count
|
|
label_table.append((histogram.name(), string_table.stringIndexes(labels)))
|
|
label_count += len(labels)
|
|
|
|
keys = histogram.keys()
|
|
key_index = 0
|
|
if len(keys) > 0:
|
|
key_index = keys_count
|
|
keys_table.append((histogram.name(), string_table.stringIndexes(keys)))
|
|
keys_count += len(keys)
|
|
|
|
stores = histogram.record_into_store()
|
|
store_index = 0
|
|
if stores == ["main"]:
|
|
# if count == 1 && offset == UINT16_MAX -> only main store
|
|
store_index = "UINT16_MAX"
|
|
else:
|
|
store_index = total_store_count
|
|
store_table.append((histogram.name(), string_table.stringIndexes(stores)))
|
|
total_store_count += len(stores)
|
|
|
|
print_array_entry(
|
|
output,
|
|
histogram,
|
|
name_index,
|
|
exp_index,
|
|
label_index,
|
|
len(labels),
|
|
key_index,
|
|
len(keys),
|
|
store_index,
|
|
len(stores),
|
|
)
|
|
print("};\n", file=output)
|
|
|
|
strtab_name = "gHistogramStringTable"
|
|
string_table.writeDefinition(output, strtab_name)
|
|
static_assert(output, "sizeof(%s) <= UINT32_MAX" % strtab_name, "index overflow")
|
|
|
|
print("\n#if defined(_MSC_VER) && !defined(__clang__)", file=output)
|
|
print("const uint32_t gHistogramLabelTable[] = {", file=output)
|
|
print("#else", file=output)
|
|
print("constexpr uint32_t gHistogramLabelTable[] = {", file=output)
|
|
print("#endif", file=output)
|
|
for name, indexes in label_table:
|
|
print("/* %s */ %s," % (name, ", ".join(map(str, indexes))), file=output)
|
|
print("};", file=output)
|
|
static_assert(
|
|
output, "sizeof(gHistogramLabelTable) <= UINT16_MAX", "index overflow"
|
|
)
|
|
|
|
print("\n#if defined(_MSC_VER) && !defined(__clang__)", file=output)
|
|
print("const uint32_t gHistogramKeyTable[] = {", file=output)
|
|
print("#else", file=output)
|
|
print("constexpr uint32_t gHistogramKeyTable[] = {", file=output)
|
|
print("#endif", file=output)
|
|
for name, indexes in keys_table:
|
|
print("/* %s */ %s," % (name, ", ".join(map(str, indexes))), file=output)
|
|
print("};", file=output)
|
|
static_assert(output, "sizeof(gHistogramKeyTable) <= UINT16_MAX", "index overflow")
|
|
|
|
store_table_name = "gHistogramStoresTable"
|
|
print("\n#if defined(_MSC_VER) && !defined(__clang__)", file=output)
|
|
print("const uint32_t {}[] = {{".format(store_table_name), file=output)
|
|
print("#else", file=output)
|
|
print("constexpr uint32_t {}[] = {{".format(store_table_name), file=output)
|
|
print("#endif", file=output)
|
|
for name, indexes in store_table:
|
|
print("/* %s */ %s," % (name, ", ".join(map(str, indexes))), file=output)
|
|
print("};", file=output)
|
|
static_assert(
|
|
output, "sizeof(%s) <= UINT16_MAX" % store_table_name, "index overflow"
|
|
)
|
|
|
|
|
|
# Write out static asserts for histogram data. We'd prefer to perform
|
|
# these checks in this script itself, but since several histograms
|
|
# (generally enumerated histograms) use compile-time constants for
|
|
# their upper bounds, we have to let the compiler do the checking.
|
|
|
|
|
|
def static_asserts_for_boolean(output, histogram):
|
|
pass
|
|
|
|
|
|
def static_asserts_for_flag(output, histogram):
|
|
pass
|
|
|
|
|
|
def static_asserts_for_count(output, histogram):
|
|
pass
|
|
|
|
|
|
def static_asserts_for_enumerated(output, histogram):
|
|
n_values = histogram.high()
|
|
static_assert(
|
|
output, "%s > 2" % n_values, "Not enough values for %s" % histogram.name()
|
|
)
|
|
|
|
|
|
def shared_static_asserts(output, histogram):
|
|
name = histogram.name()
|
|
low = histogram.low()
|
|
high = histogram.high()
|
|
n_buckets = histogram.n_buckets()
|
|
static_assert(output, "%s < %s" % (low, high), "low >= high for %s" % name)
|
|
static_assert(output, "%s > 2" % n_buckets, "Not enough values for %s" % name)
|
|
static_assert(output, "%s >= 1" % low, "Incorrect low value for %s" % name)
|
|
static_assert(
|
|
output,
|
|
"%s > %s" % (high, n_buckets),
|
|
"high must be > number of buckets for %s;"
|
|
" you may want an enumerated histogram" % name,
|
|
)
|
|
|
|
|
|
def static_asserts_for_linear(output, histogram):
|
|
shared_static_asserts(output, histogram)
|
|
|
|
|
|
def static_asserts_for_exponential(output, histogram):
|
|
shared_static_asserts(output, histogram)
|
|
|
|
|
|
def write_histogram_static_asserts(output, histograms):
|
|
print(
|
|
"""
|
|
// Perform the checks at the beginning of HistogramGet at
|
|
// compile time, so that incorrect histogram definitions
|
|
// give compile-time errors, not runtime errors.""",
|
|
file=output,
|
|
)
|
|
|
|
table = {
|
|
"boolean": static_asserts_for_boolean,
|
|
"flag": static_asserts_for_flag,
|
|
"count": static_asserts_for_count,
|
|
"enumerated": static_asserts_for_enumerated,
|
|
"categorical": static_asserts_for_enumerated,
|
|
"linear": static_asserts_for_linear,
|
|
"exponential": static_asserts_for_exponential,
|
|
}
|
|
|
|
target_os = buildconfig.substs["OS_TARGET"]
|
|
for histogram in histograms:
|
|
kind = histogram.kind()
|
|
if not histogram.record_on_os(target_os):
|
|
continue
|
|
|
|
if kind not in table:
|
|
raise Exception(
|
|
'Unknown kind "%s" for histogram "%s".' % (kind, histogram.name())
|
|
)
|
|
fn = table[kind]
|
|
fn(output, histogram)
|
|
|
|
|
|
def write_histogram_ranges(output, histograms):
|
|
# This generates static data to avoid costly initialization of histograms
|
|
# (especially exponential ones which require log and exp calls) at runtime.
|
|
# The format must exactly match that required in histogram.cc, which is
|
|
# 0, buckets..., INT_MAX. Additionally, the list ends in a 0 to aid asserts
|
|
# that validate that the length of the ranges list is correct.U cache miss.
|
|
print("#if defined(_MSC_VER) && !defined(__clang__)", file=output)
|
|
print("const int gHistogramBucketLowerBounds[] = {", file=output)
|
|
print("#else", file=output)
|
|
print("constexpr int gHistogramBucketLowerBounds[] = {", file=output)
|
|
print("#endif", file=output)
|
|
|
|
# Print the dummy buckets for expired histograms, and set the offset to match.
|
|
print("0,1,2,INT_MAX,", file=output)
|
|
offset = 4
|
|
ranges_offsets = {}
|
|
|
|
for histogram in histograms:
|
|
ranges = tuple(histogram.ranges())
|
|
if ranges not in ranges_offsets:
|
|
ranges_offsets[ranges] = offset
|
|
# Suffix each ranges listing with INT_MAX, to match histogram.cc's
|
|
# expected format.
|
|
offset += len(ranges) + 1
|
|
print(",".join(map(str, ranges)), ",INT_MAX,", file=output)
|
|
print("0};", file=output)
|
|
|
|
if offset > 32767:
|
|
raise Exception("Histogram offsets exceeded maximum value for an int16_t.")
|
|
|
|
target_os = buildconfig.substs["OS_TARGET"]
|
|
print("#if defined(_MSC_VER) && !defined(__clang__)", file=output)
|
|
print("const int16_t gHistogramBucketLowerBoundIndex[] = {", file=output)
|
|
print("#else", file=output)
|
|
print("constexpr int16_t gHistogramBucketLowerBoundIndex[] = {", file=output)
|
|
print("#endif", file=output)
|
|
for histogram in histograms:
|
|
if histogram.record_on_os(target_os):
|
|
our_offset = ranges_offsets[tuple(histogram.ranges())]
|
|
print("%d," % our_offset, file=output)
|
|
|
|
print("};", file=output)
|
|
|
|
|
|
def main(output, *filenames):
|
|
try:
|
|
histograms = list(parse_histograms.from_files(filenames))
|
|
except ParserError as ex:
|
|
print("\nError processing histograms:\n" + str(ex) + "\n")
|
|
sys.exit(1)
|
|
|
|
print(banner, file=output)
|
|
write_histogram_table(output, histograms)
|
|
write_histogram_ranges(output, histograms)
|
|
write_histogram_static_asserts(output, histograms)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.stdout, *sys.argv[1:])
|