Skip to content

Commit 99e457a

Browse files
authored
Enable metrics collection for CI builds. (#211)
The change instruments fireci with opencensus stats backed by Stackdriver Monitoring. Metrics collection is controlled via "FIREBASE_ENABLE_METRICS" environment or the "--enable-metrics" command line flag. The change additionally propagates the stats Tag context to gradle for future use.
1 parent 58b2928 commit 99e457a

File tree

6 files changed

+200
-8
lines changed

6 files changed

+200
-8
lines changed

ci/fireci/fireci/commands.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
from . import gradle
1919
from . import ci_command
20+
from . import stats
2021

2122

2223
@click.argument('task', required=True, nargs=-1)

ci/fireci/fireci/emulator.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import subprocess
2020
import time
2121

22+
from . import stats
23+
2224
_logger = logging.getLogger('fireci.emulator')
2325

2426
EMULATOR_BINARY = 'emulator'
@@ -65,6 +67,7 @@ def __init__(
6567
self._wait_for_device_stdin = wait_for_device_stdin
6668
self._logcat_stdin = logcat_stdin
6769

70+
@stats.measure_call("emulator_startup")
6871
def __enter__(self):
6972
_logger.info('Starting avd "{}..."'.format(self._name))
7073
self._process = subprocess.Popen(
@@ -86,6 +89,7 @@ def __enter__(self):
8689
stdout=self._adb_log,
8790
)
8891

92+
@stats.measure_call("emulator_shutdown")
8993
def __exit__(self, exception_type, exception_value, traceback):
9094
_logger.info('Shutting down avd "{}"...'.format(self._name))
9195
self._kill(self._process)

ci/fireci/fireci/gradle.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
import subprocess
1818
import sys
1919

20+
from . import stats
21+
2022
_logger = logging.getLogger('fireci.gradle')
2123

2224
ADB_INSTALL_TIMEOUT = '5'
@@ -34,6 +36,7 @@ def run(*args, gradle_opts='', workdir=None):
3436
new_env['GRADLE_OPTS'] = gradle_opts
3537
new_env[
3638
'ADB_INSTALL_TIMEOUT'] = ADB_INSTALL_TIMEOUT # 5 minutes, rather than 2 minutes
39+
stats.propagate_context_into(new_env)
3740

3841
command = ['./gradlew'] + list(args)
3942
_logger.info('Executing gradle command: "%s" in directory: "%s"',

ci/fireci/fireci/internal.py

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import shutil
2323

2424
from . import emulator
25+
from . import stats
2526

2627
_logger = logging.getLogger('fireci')
2728

@@ -104,6 +105,11 @@ class _CommonOptions:
104105
default='adb',
105106
help='Specifies the name/full path to the adb binary.',
106107
)
108+
@click.option(
109+
'--enable-metrics',
110+
is_flag=True,
111+
envvar='FIREBASE_ENABLE_METRICS',
112+
help='Enables metrics collection for various build stages.')
107113
@_pass_options
108114
def main(options, **kwargs):
109115
"""Main command group.
@@ -112,6 +118,8 @@ def main(options, **kwargs):
112118
"""
113119
for k, v in kwargs.items():
114120
setattr(options, k, v)
121+
if options.enable_metrics:
122+
stats.configure()
115123

116124

117125
def ci_command(name=None):
@@ -127,18 +135,20 @@ def ci_command(name=None):
127135
"""
128136

129137
def ci_command(f):
138+
actual_name = f.__name__ if name is None else name
130139

131-
@main.command(name=f.__name__ if name is None else name, help=f.__doc__)
140+
@main.command(name=actual_name, help=f.__doc__)
132141
@_pass_options
133142
@click.pass_context
134143
def new_func(ctx, options, *args, **kwargs):
135-
with _artifact_handler(options.artifact_target_dir,
136-
options.artifact_patterns), _emulator_handler(
137-
options.with_emulator,
138-
options.artifact_target_dir,
139-
name=options.emulator_name,
140-
emulator_binary=options.emulator_binary,
141-
adb_binary=options.adb_binary):
144+
with stats.measure("cicmd:" + actual_name), _artifact_handler(
145+
options.artifact_target_dir,
146+
options.artifact_patterns), _emulator_handler(
147+
options.with_emulator,
148+
options.artifact_target_dir,
149+
name=options.emulator_name,
150+
emulator_binary=options.emulator_binary,
151+
adb_binary=options.adb_binary):
142152
return ctx.invoke(f, *args, **kwargs)
143153

144154
return functools.update_wrapper(new_func, f)

ci/fireci/fireci/stats.py

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import base64
16+
import contextlib
17+
import copy
18+
import functools
19+
import google.auth
20+
import google.auth.exceptions
21+
import logging
22+
import os
23+
import time
24+
25+
from opencensus import tags
26+
from opencensus.stats import aggregation
27+
from opencensus.stats import measure
28+
from opencensus.stats import stats
29+
from opencensus.stats import view
30+
from opencensus.stats.exporters import stackdriver_exporter
31+
from opencensus.stats.exporters.base import StatsExporter
32+
from opencensus.tags import execution_context
33+
from opencensus.tags.propagation import binary_serializer
34+
35+
_logger = logging.getLogger('fireci.stats')
36+
STATS = stats.Stats()
37+
38+
_m_latency = measure.MeasureFloat("latency", "The latency in milliseconds",
39+
"ms")
40+
_m_success = measure.MeasureInt("success", "Indicated success or failure.", "1")
41+
42+
_key_stage = tags.TagKey("stage")
43+
44+
_TAGS = [
45+
_key_stage,
46+
tags.TagKey("repo_owner"),
47+
tags.TagKey("repo_name"),
48+
tags.TagKey("pull_number"),
49+
tags.TagKey("job_name"),
50+
]
51+
52+
_METRICS_ENABLED = False
53+
54+
55+
class StdoutExporter(StatsExporter):
56+
"""Fallback exporter in case stackdriver cannot be configured."""
57+
58+
def on_register_view(self, view):
59+
pass
60+
61+
def emit(self, view_datas):
62+
_logger.info("emit %s", self.repr_data(view_datas))
63+
64+
def export(self, view_data):
65+
_logger.info("export %s", self._repr_data(view_data))
66+
67+
@staticmethod
68+
def _repr_data(view_data):
69+
return [
70+
"ViewData<view={}, start={}, end={}>".format(d.view, d.start_time,
71+
d.end_time)
72+
for d in view_data
73+
]
74+
75+
76+
def _new_exporter():
77+
"""
78+
Initializes a metrics exporter.
79+
80+
Tries to initialize a Stackdriver exporter, falls back to StdoutExporter.
81+
"""
82+
try:
83+
_, project_id = google.auth.default()
84+
return stackdriver_exporter.new_stats_exporter(
85+
stackdriver_exporter.Options(project_id=project_id, resource='global'))
86+
except google.auth.exceptions.DefaultCredentialsError:
87+
_logger.exception("Using stdout exporter")
88+
return StdoutExporter()
89+
90+
91+
def configure():
92+
"""Globally enables metrics collection."""
93+
global _METRICS_ENABLED
94+
if _METRICS_ENABLED:
95+
return
96+
_METRICS_ENABLED = True
97+
98+
STATS.view_manager.register_exporter(_new_exporter())
99+
latency_view = view.View(
100+
"fireci/latency", "Latency of fireci execution stages", _TAGS, _m_latency,
101+
aggregation.LastValueAggregation())
102+
success_view = view.View(
103+
"fireci/success", "Success indication of fireci execution stages", _TAGS,
104+
_m_success, aggregation.LastValueAggregation())
105+
STATS.view_manager.register_view(latency_view)
106+
STATS.view_manager.register_view(success_view)
107+
108+
context = tags.TagMap()
109+
for tag in _TAGS:
110+
if tag.upper() in os.environ:
111+
context.insert(tag, tags.TagValue(os.environ[tag.upper()]))
112+
113+
execution_context.set_current_tag_map(context)
114+
115+
116+
@contextlib.contextmanager
117+
def _measure(name):
118+
tmap = copy.deepcopy(execution_context.get_current_tag_map())
119+
tmap.insert(_key_stage, name)
120+
start = time.time()
121+
try:
122+
yield
123+
except:
124+
mmap = STATS.stats_recorder.new_measurement_map()
125+
mmap.measure_int_put(_m_success, 0)
126+
mmap.record(tmap)
127+
raise
128+
129+
elapsed = (time.time() - start) * 1000
130+
mmap = STATS.stats_recorder.new_measurement_map()
131+
mmap.measure_float_put(_m_latency, elapsed)
132+
mmap.measure_int_put(_m_success, 1)
133+
mmap.record(tmap)
134+
_logger.info("%s took %sms", name, elapsed)
135+
136+
137+
@contextlib.contextmanager
138+
def measure(name):
139+
"""Context manager that measures the time it took for a block of code to execute."""
140+
if not _METRICS_ENABLED:
141+
yield
142+
return
143+
with _measure(name):
144+
yield
145+
146+
147+
def measure_call(name):
148+
"""Function decorator that measures the time it took to execute the target function."""
149+
150+
def decorator(f):
151+
152+
def decorated(*args, **kwargs):
153+
with measure(name):
154+
f(*args, **kwargs)
155+
156+
functools.update_wrapper(decorated, f)
157+
return decorated
158+
159+
return decorator
160+
161+
162+
def propagate_context_into(data_dict):
163+
"""Propagates Tag context into a dictionary."""
164+
if not _METRICS_ENABLED:
165+
return
166+
value = binary_serializer.BinarySerializer().to_byte_array(
167+
execution_context.get_current_tag_map())
168+
data_dict['OPENCENSUS_STATS_CONTEXT'] = base64.b64encode(value)

ci/fireci/setup.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,14 @@
2424
setup(
2525
name='fireci',
2626
version='0.1',
27+
# this is a temporary measure until opencensus 0.2 release is out.
28+
dependency_links=[
29+
'https://github.com/census-instrumentation/opencensus-python/tarball/master#egg=opencensus'
30+
],
2731
install_requires=[
2832
'click==7.0',
33+
'opencensus',
34+
'google-cloud-monitoring==0.31.1',
2935
],
3036
packages=find_packages(exclude=['tests']),
3137
entry_points={

0 commit comments

Comments
 (0)