1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
|
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Outputter to generate Rust code for metrics.
"""
import enum
import json
import sys
import jinja2
from glean_parser import util
from glean_parser.metrics import Rate
from util import type_ids_and_categories
from js import ID_BITS, PING_INDEX_BITS
RUNTIME_METRIC_BIT = ID_BITS - 1
RUNTIME_PING_BIT = PING_INDEX_BITS - 1
# The list of all args to CommonMetricData.
# No particular order is required, but I have these in common_metric_data.rs
# order just to be organized.
# Note that this is util.common_metric_args + "dynamic_label"
common_metric_data_args = [
"name",
"category",
"send_in_pings",
"lifetime",
"disabled",
"dynamic_label",
]
# List of all metric-type-specific args that JOG understands.
known_extra_args = [
"time_unit",
"memory_unit",
"allowed_extra_keys",
"reason_codes",
"range_min",
"range_max",
"bucket_count",
"histogram_type",
"numerators",
"ordered_labels",
]
# List of all ping-specific args that JOG undertsands.
known_ping_args = [
"name",
"include_client_id",
"send_if_empty",
"precise_timestamps",
"include_info_sections",
"enabled",
"schedules_pings",
"reason_codes",
]
def ensure_jog_support_for_args():
"""
glean_parser or the Glean SDK might add new metric/ping args.
To ensure JOG doesn't fall behind in support,
we check the list of JOG-supported args vs glean_parser's.
We fail the build if glean_parser has one or more we haven't seen before.
"""
unknown_args = set(util.extra_metric_args) - set(known_extra_args)
unknown_args |= set(util.ping_args) - set(known_ping_args)
if len(unknown_args):
print(f"Unknown glean_parser args {unknown_args}")
print("JOG must be updated to support the new args")
sys.exit(1)
def load_monkeypatches():
"""
Monkeypatch jinja template loading because we're not glean_parser.
We're glean_parser_ext.
"""
# Monkeypatch util.get_jinja2_template to find templates nearby
def get_local_template(template_name, filters=()):
env = jinja2.Environment(
loader=jinja2.PackageLoader("rust", "templates"),
trim_blocks=True,
lstrip_blocks=True,
)
env.filters["camelize"] = util.camelize
env.filters["Camelize"] = util.Camelize
for filter_name, filter_func in filters:
env.filters[filter_name] = filter_func
return env.get_template(template_name)
util.get_jinja2_template = get_local_template
def output_factory(objs, output_fd, options={}):
"""
Given a tree of objects, output Rust code to the file-like object `output_fd`.
Specifically, Rust code that can generate Rust metrics instances.
:param objs: A tree of objects (metrics and pings) as returned from
`parser.parse_objects`.
:param output_fd: Writeable file to write the output to.
:param options: options dictionary, presently unused.
"""
ensure_jog_support_for_args()
load_monkeypatches()
# Get the metric type ids. Must be the same ids generated in js.py
metric_types, categories = type_ids_and_categories(objs)
template = util.get_jinja2_template(
"jog_factory.jinja2",
filters=(("snake_case", util.snake_case),),
)
output_fd.write(
template.render(
all_objs=objs,
common_metric_data_args=common_metric_data_args,
extra_args=util.extra_args,
metric_types=metric_types,
runtime_metric_bit=RUNTIME_METRIC_BIT,
runtime_ping_bit=RUNTIME_PING_BIT,
ID_BITS=ID_BITS,
)
)
output_fd.write("\n")
def camel_to_snake(s):
assert "_" not in s, "JOG doesn't encode metric typenames with underscores"
return "".join(["_" + c.lower() if c.isupper() else c for c in s]).lstrip("_")
def output_file(objs, output_fd, options={}):
"""
Given a tree of objects, output them to the file-like object `output_fd`.
Specifically, in a format that describes all the metrics and pings defined in objs.
:param objs: A tree of objects (metrics and pings) as returned from
`parser.parse_objects`.
Presently a dictionary with keys of literals "pings" and "tags"
as well as one key per metric category mapped to lists of
pings, tags, and metrics (respecitvely)
:param output_fd: Writeable file to write the output to.
:param options: options dictionary, presently unused.
"""
ensure_jog_support_for_args()
jog_data = {"pings": [], "metrics": {}}
if "tags" in objs:
del objs["tags"] # JOG has no use for tags.
pings = objs["pings"]
del objs["pings"]
for ping in pings.values():
ping_arg_list = []
for arg in known_ping_args:
if hasattr(ping, arg):
ping_arg_list.append(getattr(ping, arg))
jog_data["pings"].append(ping_arg_list)
def encode(value):
if isinstance(value, enum.Enum):
return value.name
if isinstance(value, Rate): # `numerators` for an external Denominator metric
args = []
for arg_name in common_metric_data_args[:-1]:
args.append(getattr(value, arg_name))
# These are deserialized as CommonMetricData.
# CMD have a final param JOG never uses: `dynamic_label`
# It's optional, so we should be able to omit it, but we'd need to
# annotate it with #[serde(default)]... so here we add the sixth
# param as None.
args.append(None)
return args
return json.dumps(value)
for category, metrics in objs.items():
dict_cat = jog_data["metrics"].setdefault(category, [])
for metric in metrics.values():
metric_arg_list = [camel_to_snake(metric.__class__.__name__)]
for arg in common_metric_data_args[:-1]:
if arg in ["category"]:
continue # We don't include the category in each metric.
metric_arg_list.append(getattr(metric, arg))
extra = {}
for arg in known_extra_args:
if hasattr(metric, arg):
extra[arg] = getattr(metric, arg)
if len(extra):
metric_arg_list.append(extra)
dict_cat.append(metric_arg_list)
# TODO: Measure the speed gain of removing `indent=2`
json.dump(jog_data, output_fd, sort_keys=True, default=encode, indent=2)
|