Implementation of HistogramVec by providing LuaC bindings to Rust module prometheus

.gitignore

@@ -18,3 +18,4 @@ doc/locale/en/
 
 build.luarocks
 packpack
+metrics-rs/target

CMakeLists.txt

@@ -2,6 +2,12 @@ cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
 
 project(metrics NONE)
 
+if(NOT DEFINED TARANTOOL_INSTALL_LIBDIR)
+    set(TARANTOOL_INSTALL_LIBDIR "${PROJECT_SOURCE_DIR}/.rocks/lib/tarantool")
+endif()
+
+add_subdirectory(metrics-rs)
+
 ## Install ####################################################################
 ###############################################################################
 

Makefile

@@ -44,6 +44,11 @@ update-pot:
 update-po:
 	sphinx-intl update -p doc/locale/en/ -d doc/locale/ -l "ru"
 
+bench: .rocks
+	$(TTCTL) rocks install --server https://moonlibs.org luabench 0.3.0
+	$(TTCTL) rocks install --server https://luarocks.org argparse 0.7.1
+	.rocks/bin/luabench -d 3s
+
 .PHONY: clean
 clean:
 	rm -rf .rocks

benchmarks/histogram_bench.lua

@@ -0,0 +1,195 @@
+local metrics = require 'metrics'
+local luahist = assert(metrics.histogram, 'no histogram')
+local rusthist = assert(metrics.histogram_vec, 'no histogram_vec')
+
+local M = {}
+
+do
+    local lh = luahist('no_labels', 'histogram')
+    local rh = rusthist('rust_no_labels', 'histogram')
+    local random = math.random
+
+    ---@param b luabench.B
+    function M.bench_001_no_labels_001_observe(b)
+        b:run("histogram:observe", function(sb)
+            for _ = 1, sb.N do lh:observe(random()) end
+        end)
+        b:run("histogram_vec:observe", function(sb)
+            for _ = 1, sb.N do rh:observe(random()) end
+        end)
+    end
+
+    ---@param b luabench.B
+    function M.bench_001_no_labels_002_collect(b)
+        b:run("histogram:collect", function(sb)
+            for _ = 1, sb.N do lh:collect() end
+        end)
+        b:run("histogram_vec:collect", function(sb)
+            for _ = 1, sb.N do rh:collect() end
+        end)
+        b:run("histogram_vec:collect_str", function(sb)
+            for _ = 1, sb.N do rh:collect_str() end
+        end)
+    end
+end
+
+do
+    local lh = luahist('one_label', 'histogram')
+    local rh = rusthist('rust_one_label', 'histogram', {'status'})
+    local random = math.random
+
+    ---@param b luabench.B
+    function M.bench_002_one_label_001_observe(b)
+        b:run("histogram:observe", function(sb)
+            for _ = 1, sb.N do
+                local x = random()
+                lh:observe(x, {status = x < 0.99 and 'ok' or 'fail'})
+            end
+        end)
+        b:run("histogram_vec:observe", function(sb)
+            for _ = 1, sb.N do
+                local x = random()
+                rh:observe(x, {status = x < 0.99 and 'ok' or 'fail'})
+            end
+        end)
+    end
+
+    ---@param b luabench.B
+    function M.bench_002_one_label_002_collect(b)
+        b:run("histogram:collect", function(sb)
+            for _ = 1, sb.N do lh:collect() end
+        end)
+        b:run("histogram_vec:collect", function(sb)
+            for _ = 1, sb.N do rh:collect() end
+        end)
+        b:run("histogram_vec:collect_str", function(sb)
+            for _ = 1, sb.N do rh:collect_str() end
+        end)
+    end
+end
+
+do
+    local lh = luahist('two_labels', 'histogram')
+    local rh = rusthist('rust_two_labels', 'histogram', {'status', 'method'})
+    local random = math.random
+
+    local methods = {'GET', 'POST', 'PUT', 'DELETE'}
+
+    ---@param b luabench.B
+    function M.bench_002_two_labels_001_observe(b)
+        b:run("histogram:observe", function(sb)
+            for _ = 1, sb.N do
+                local x = random()
+                lh:observe(x, {status = x < 0.99 and 'ok' or 'fail', method = methods[random(1, 4)]})
+            end
+        end)
+        b:run("histogram_vec:observe", function(sb)
+            for _ = 1, sb.N do
+                local x = random()
+                rh:observe(x, {status = x < 0.99 and 'ok' or 'fail', method = methods[random(1, 4)]})
+            end
+        end)
+    end
+
+    ---@param b luabench.B
+    function M.bench_002_two_labels_002_collect(b)
+        b:run("histogram:collect", function(sb)
+            for _ = 1, sb.N do lh:collect() end
+        end)
+        b:run("histogram_vec:collect", function(sb)
+            for _ = 1, sb.N do rh:collect() end
+        end)
+        b:run("histogram_vec:collect_str", function(sb)
+            for _ = 1, sb.N do rh:collect_str() end
+        end)
+    end
+end
+
+do
+    local lh = luahist('three_labels', 'histogram')
+    local rh = rusthist('rust_three_labels', 'histogram', {'status', 'method', 'func'})
+    local random = math.random
+
+    local methods = {'GET', 'POST', 'PUT', 'DELETE'}
+    local funcs = {
+        'api.pet.find_by_status',
+        'api.pet.find_by_id',
+        'api.pet.get',
+        'api.pet.find_by_tags',
+        'api.pet.post',
+        'api.pet.put',
+        'api.pet.delete',
+        'api.store.inventory.get',
+        'api.store.order.post',
+        'api.store.order.get',
+        'api.store.order.delete',
+        'api.user.get',
+        'api.user.put',
+        'api.user.delete',
+        'api.user.post',
+        'api.user.login',
+        'api.user.logout',
+    }
+    local n_funcs = #funcs
+
+    ---@param b luabench.B
+    function M.bench_003_three_labels_001_observe(b)
+        b:run("histogram:observe", function(sb)
+            for _ = 1, sb.N do
+                local x = random()
+                lh:observe(x, {
+                    status = x < 0.99 and 'ok' or 'fail',
+                    method = methods[random(1, 4)],
+                    func = funcs[random(1, n_funcs)],
+                })
+            end
+        end)
+        b:run("histogram_vec:observe", function(sb)
+            for _ = 1, sb.N do
+                local x = random()
+                rh:observe(x, {
+                    status = x < 0.99 and 'ok' or 'fail',
+                    method = methods[random(1, 4)],
+                    func = funcs[random(1, n_funcs)],
+                })
+            end
+        end)
+    end
+
+    ---@param b luabench.B
+    function M.bench_003_three_labels_002_collect(b)
+        b:run("histogram:collect", function(sb)
+            for _ = 1, sb.N do lh:collect() end
+        end)
+        b:run("histogram_vec:collect", function(sb)
+            for _ = 1, sb.N do rh:collect() end
+        end)
+        b:run("histogram_vec:collect_str", function(sb)
+            for _ = 1, sb.N do rh:collect_str() end
+        end)
+    end
+end
+
+do
+    local rs = require('override.metrics.rs')
+    function M.bench_004_rust_gather(b)
+        for _ = 1, b.N do
+            rs.gather()
+        end
+    end
+
+    function M.bench_004_lua_gather(b)
+        for _ = 1, b.N do
+            local result = {}
+            for _, c in pairs(metrics.registry.collectors) do
+                if not c.collect_str then
+                    for _, obs in ipairs(c:collect()) do
+                        table.insert(result, obs)
+                    end
+                end
+            end
+        end
+    end
+end
+
+return M

metrics-rs/CMakeLists.txt

@@ -0,0 +1,23 @@
+cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
+project(metrics-rs NONE)
+
+if (NOT DEFINED CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE "Release")
+endif()
+if (NOT DEFINED TARANTOOL_INSTALL_LIBDIR)
+    message(SEND_ERROR "TARANTOOL_INSTALL_LIBDIR is not defined")
+endif()
+
+set(RUST_BUILD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/target/${CMAKE_BUILD_TYPE})
+
+find_program(CARGO cargo)
+add_custom_target(metrics-rs ALL
+    COMMAND ${CARGO} build --profile "${CMAKE_BUILD_TYPE}"
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
+install(
+    FILES ${RUST_BUILD_DIR}/libmetrics_rs${CMAKE_SHARED_LIBRARY_SUFFIX}
+    RENAME metrics_rs${CMAKE_SHARED_LIBRARY_SUFFIX}
+    DESTINATION ${TARANTOOL_INSTALL_LIBDIR}
+)

metrics-rs/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "metrics_rs"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+mlua = { version = "0.10.3", features = ["luajit", "module", "serialize"] }
+prometheus = "0.13.4"
+serde = { version = "1.0.218", features = ["derive"] }
+
+[lib]
+crate-type = ["cdylib"]
+
+[profile.RelWithDebInfo]
+inherits = "release"
+debug = true
+
+[profile.Release]
+inherits = "release"

metrics-rs/src/histogram_vec.rs

@@ -0,0 +1,264 @@
+use std::collections::HashMap;
+
+use mlua::prelude::*;
+use prometheus::core::Collector;
+use prometheus::proto;
+use serde::{Deserialize, Serialize};
+
+use crate::registry;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct LuaHistogramOpts {
+    pub name: String,
+    pub help: String,
+    pub buckets: Option<Vec<f64>>,
+}
+
+#[derive(Debug)]
+pub struct LuaHistogramVec {
+    pub histogram_vec: Box<prometheus::HistogramVec>,
+    pub name: String,
+}
+
+impl LuaHistogramVec {
+    pub fn new(opts: LuaHistogramOpts, label_names: LuaTable) -> LuaResult<Self> {
+        let name = opts.name.clone();
+        let mut hopts = prometheus::HistogramOpts::new(opts.name, opts.help);
+        if let Some(buckets) = opts.buckets {
+            hopts = hopts.buckets(buckets);
+        }
+
+        let label_names: Vec<String> = label_names
+            .pairs()
+            .map(|kv| -> LuaResult<String> {
+                let (_, v): (LuaValue, LuaString) = kv?;
+                Ok(v.to_string_lossy())
+            })
+            .collect::<LuaResult<Vec<_>>>()?;
+
+        let label_names = label_names.iter().map(|s| &**s).collect::<Vec<&str>>();
+
+        let hist = Box::new(
+            prometheus::HistogramVec::new(hopts, &label_names).map_err(mlua::Error::external)?,
+        );
+
+        registry::register(hist.clone()).map_err(mlua::Error::external)?;
+
+        Ok(Self {
+            histogram_vec: hist,
+            name,
+        })
+    }
+
+    pub fn observe(&self, value: f64, label_values: Option<Vec<String>>) -> LuaResult<()> {
+        if let Some(lv) = label_values {
+            let label_values = lv.iter().map(|s| &**s).collect::<Vec<&str>>();
+
+            self.histogram_vec
+                .get_metric_with_label_values(&label_values)
+                .map_err(mlua::Error::external)?
+                .observe(value);
+        } else {
+            self.histogram_vec
+                .get_metric_with_label_values(&[])
+                .map_err(mlua::Error::external)?
+                .observe(value);
+        }
+
+        Ok(())
+    }
+
+    pub fn remove(&mut self, lv: Vec<String>) -> LuaResult<()> {
+        let label_values = lv.iter().map(|s| &**s).collect::<Vec<&str>>();
+
+        self.histogram_vec
+            .remove_label_values(&label_values)
+            .map_err(mlua::Error::external)
+    }
+
+    pub fn collect_str(&self, lua: &Lua) -> LuaResult<LuaString> {
+        let mfs = self.histogram_vec.collect();
+
+        let result = prometheus::TextEncoder::new()
+            .encode_to_string(&mfs)
+            .map_err(mlua::Error::external)?;
+
+        lua.create_string(result)
+    }
+
+    pub fn collect(
+        &self,
+        lua: &Lua,
+        global_values: Option<HashMap<String, String>>,
+    ) -> LuaResult<LuaTable> {
+        let result = lua.create_table()?;
+        let mfs = self.histogram_vec.collect();
+
+        let pairs: Option<Vec<proto::LabelPair>> = global_values.map(|ref hmap| {
+            hmap.iter()
+                .map(|(k, v)| {
+                    let mut label = proto::LabelPair::default();
+                    label.set_name(k.to_string());
+                    label.set_value(v.to_string());
+                    label
+                })
+                .collect()
+        });
+
+        for mut mf in mfs.into_iter() {
+            if mf.get_metric().is_empty() {
+                continue;
+            }
+
+            let metric_name = mf.get_name();
+            let lbucket_name = lua.create_string(metric_name.to_string() + "_bucket")?;
+            let lsum_name = lua.create_string(metric_name.to_string() + "_sum")?;
+            let lcount_name = lua.create_string(metric_name.to_string() + "_count")?;
+
+            // append global labels if needed
+            if let Some(pairs) = &pairs {
+                for metric in mf.mut_metric().iter_mut() {
+                    let mut labels: Vec<_> = metric.take_label().into();
+                    labels.append(&mut pairs.clone());
+                    metric.set_label(labels.into());
+                }
+            }
+
+            for metric in mf.get_metric().iter() {
+                let time = metric.get_timestamp_ms() * 1000;
+
+                // we convert labels into pair of lua strings
+                let mut labels: Vec<_> = Vec::with_capacity(metric.get_label().len());
+
+                for pair in metric.get_label().iter() {
+                    let k = pair.get_name();
+                    let v = pair.get_value();
+
+                    let k = lua.create_string(k)?;
+                    let v = lua.create_string(v)?;
+
+                    labels.push((k, v));
+                }
+
+                let h = metric.get_histogram();
+
+                let blabels = lua.create_table()?;
+                for pair in labels.iter() {
+                    blabels.set(&pair.0, &pair.1)?
+                }
+
+                // firstly collect count
+                let lmetric = lua.create_table_with_capacity(0, 4)?;
+                lmetric.set("value", h.get_sample_count())?;
+                lmetric.set("metric_name", &lcount_name)?;
+                lmetric.set("timestamp", time)?;
+                lmetric.set("label_pairs", &blabels)?; // blabels are shared between _count and _sum
+                result.raw_push(lmetric)?;
+
+                // secondly collect sum
+                let lmetric = lua.create_table_with_capacity(0, 4)?;
+                lmetric.set("metric_name", &lsum_name)?;
+                lmetric.set("timestamp", time)?;
+                lmetric.set("value", h.get_sample_sum())?;
+                lmetric.set("label_pairs", &blabels)?; // blabels are shared between _count and _sum
+                result.raw_push(lmetric)?;
+
+                // lastly collect buckets
+                let mut inf_seen = false;
+                for b in h.get_bucket() {
+                    let lmetric = lua.create_table_with_capacity(0, 4)?;
+                    lmetric.set("metric_name", &lbucket_name)?;
+                    lmetric.set("value", b.get_cumulative_count())?;
+                    lmetric.set("timestamp", time)?;
+
+                    // upper_bound should be used as label
+                    let upper_bound = b.get_upper_bound();
+
+                    let blabels = lua.create_table()?;
+                    for pair in labels.iter() {
+                        blabels.set(&pair.0, &pair.1)?
+                    }
+                    blabels.set("le", upper_bound)?;
+
+                    lmetric.set("label_pairs", blabels)?;
+                    result.raw_push(lmetric)?;
+
+                    if b.get_upper_bound().is_infinite() {
+                        inf_seen = true;
+                    }
+                }
+
+                if !inf_seen {
+                    let lmetric = lua.create_table_with_capacity(0, 4)?;
+                    lmetric.set("metric_name", &lbucket_name)?;
+                    lmetric.set("value", h.get_sample_count())?;
+                    lmetric.set("timestamp", time)?;
+
+                    let blabels = lua.create_table()?;
+                    for pair in labels.iter() {
+                        blabels.set(&pair.0, &pair.1)?
+                    }
+                    blabels.set("le", "+Inf")?;
+
+                    lmetric.set("label_pairs", blabels)?;
+                    result.raw_push(lmetric)?;
+                }
+            }
+        }
+
+        Ok(result)
+    }
+}
+
+impl Drop for LuaHistogramVec {
+    fn drop(&mut self) {
+        let histogram_vec = self.histogram_vec.clone();
+        let r = registry::unregister(histogram_vec);
+        if let Some(err) = r.err() {
+            eprintln!(
+                "[ERR] Failed to unregister LuaHistogramVec: {}: {}",
+                self.name, err
+            );
+        }
+    }
+}
+
+impl LuaUserData for LuaHistogramVec {
+    fn add_methods<M: LuaUserDataMethods<Self>>(methods: &mut M) {
+        let tostring = |lua: &Lua, this: &LuaHistogramVec, ()| -> LuaResult<LuaString> {
+            let msg = format!("HistogramVec<{}>", this.name);
+            let s = lua.create_string(msg)?;
+            Ok(s)
+        };
+        methods.add_meta_method("__serialize", tostring);
+        methods.add_meta_method("__tostring", tostring);
+
+        methods.add_method("collect_str", |lua: &Lua, this: &Self, (): ()| {
+            this.collect_str(lua)
+        });
+
+        methods.add_method(
+            "collect",
+            |lua: &Lua, this: &Self, global_values: Option<HashMap<String, String>>| {
+                this.collect(lua, global_values)
+            },
+        );
+
+        methods.add_method_mut("remove", |_, this: &mut Self, label_values: _| {
+            this.remove(label_values)
+        });
+
+        methods.add_method(
+            "observe",
+            |_, this: &Self, (value, label_values): (_, _)| this.observe(value, label_values),
+        );
+
+        methods.add_method("unregister", |_, this: &Self, (): ()| {
+            let r = registry::unregister(this.histogram_vec.clone());
+            if let Some(err) = r.err() {
+                return Err(mlua::Error::external(err));
+            }
+            Ok(())
+        })
+    }
+}

metrics-rs/src/lib.rs

@@ -0,0 +1,39 @@
+use mlua::prelude::*;
+use std::collections::HashMap;
+
+mod histogram_vec;
+mod registry;
+use histogram_vec::{LuaHistogramOpts, LuaHistogramVec};
+
+// creates new HistogramVec with given label_names
+fn new_histogram_vec(lua: &Lua, (opts, names): (LuaValue, LuaTable)) -> LuaResult<LuaHistogramVec> {
+    let opts: LuaHistogramOpts = lua.from_value(opts)?;
+    LuaHistogramVec::new(opts, names).map_err(LuaError::external)
+}
+
+// gathers all metrics registered in default prometheus::Registry
+fn gather(lua: &Lua, (): ()) -> LuaResult<LuaString> {
+    let mfs = registry::gather();
+
+    let result = prometheus::TextEncoder::new()
+        .encode_to_string(&mfs)
+        .map_err(mlua::Error::external)?;
+
+    lua.create_string(result)
+}
+
+fn set_labels(_: &Lua, global_labels: HashMap<String, String>) -> LuaResult<()> {
+    registry::set_labels(global_labels);
+    Ok(())
+}
+
+#[mlua::lua_module]
+pub fn metrics_rs(lua: &Lua) -> LuaResult<LuaTable> {
+    let r = lua.create_table()?;
+
+    r.set("new_histogram_vec", lua.create_function(new_histogram_vec)?)?;
+    r.set("gather", lua.create_function(gather)?)?;
+    r.set("set_labels", lua.create_function(set_labels)?)?;
+
+    Ok(r)
+}

metrics-rs/src/registry.rs

@@ -0,0 +1,220 @@
+use mlua::prelude::*;
+use prometheus::core::Collector;
+use prometheus::proto;
+use std::cell::RefCell;
+use std::collections::btree_map::Entry as BEntry;
+use std::collections::hash_map::Entry as HEntry;
+use std::collections::{BTreeMap, HashMap, HashSet};
+
+/// A struct for registering Prometheus collectors, collecting their metrics, and gathering
+/// them into `MetricFamilies` for exposition.
+#[derive(Default)]
+pub struct Registry {
+    pub collectors_by_id: HashMap<u64, Box<dyn Collector>>,
+    pub dim_hashes_by_name: HashMap<String, u64>,
+    pub desc_ids: HashSet<u64>,
+    /// Optional common labels for all registered collectors.
+    pub labels: Option<HashMap<String, String>>,
+}
+
+impl Registry {
+    pub fn set_labels(&mut self, labels: HashMap<String, String>) {
+        self.labels = Some(labels);
+    }
+    pub fn register(&mut self, c: Box<dyn Collector>) -> LuaResult<()> {
+        let mut desc_id_set = HashSet::new();
+        let mut collector_id: u64 = 0;
+
+        for desc in c.desc() {
+            // Is the desc_id unique?
+            // (In other words: Is the fqName + constLabel combination unique?)
+            if self.desc_ids.contains(&desc.id) {
+                return Err(mlua::Error::external(prometheus::Error::AlreadyReg));
+            }
+
+            if let Some(hash) = self.dim_hashes_by_name.get(&desc.fq_name) {
+                if *hash != desc.dim_hash {
+                    return Err(mlua::Error::external(format!(
+                        "a previously registered descriptor with the \
+                         same fully-qualified name as {:?} has \
+                         different label names or a different help \
+                         string",
+                        desc
+                    )));
+                }
+            }
+
+            self.dim_hashes_by_name
+                .insert(desc.fq_name.clone(), desc.dim_hash);
+
+            // If it is not a duplicate desc in this collector, add it to
+            // the collector_id.
+            if desc_id_set.insert(desc.id) {
+                // The set did not have this value present, true is returned.
+                collector_id = collector_id.wrapping_add(desc.id);
+            } else {
+                // The set did have this value present, false is returned.
+                //
+                return Err(mlua::Error::external(format!(
+                    "a duplicate descriptor within the same \
+                     collector the same fully-qualified name: {:?}",
+                    desc.fq_name
+                )));
+            }
+        }
+
+        match self.collectors_by_id.entry(collector_id) {
+            HEntry::Vacant(vc) => {
+                self.desc_ids.extend(desc_id_set);
+                vc.insert(c);
+                Ok(())
+            }
+            HEntry::Occupied(_) => Err(mlua::Error::external(prometheus::Error::AlreadyReg)),
+        }
+    }
+
+    pub fn unregister(&mut self, c: Box<dyn Collector>) -> LuaResult<()> {
+        let mut id_set = Vec::new();
+        let mut collector_id: u64 = 0;
+        for desc in c.desc() {
+            if !id_set.iter().any(|id| *id == desc.id) {
+                id_set.push(desc.id);
+                collector_id = collector_id.wrapping_add(desc.id);
+            }
+            let _ = self.dim_hashes_by_name.remove(&desc.fq_name);
+        }
+
+        if self.collectors_by_id.remove(&collector_id).is_none() {
+            return Err(mlua::Error::external(format!(
+                "collector {:?} is not registered",
+                c.desc()
+            )));
+        }
+
+        for id in id_set {
+            self.desc_ids.remove(&id);
+        }
+
+        // dim_hashes_by_name is left untouched as those must be consistent
+        // throughout the lifetime of a program.
+        Ok(())
+    }
+
+    pub fn gather(&self) -> Vec<proto::MetricFamily> {
+        let mut mf_by_name = BTreeMap::new();
+
+        for c in self.collectors_by_id.values() {
+            let mfs = c.collect();
+            for mut mf in mfs {
+                // Prune empty MetricFamilies.
+                if mf.get_metric().is_empty() {
+                    continue;
+                }
+
+                let name = mf.get_name().to_owned();
+                match mf_by_name.entry(name) {
+                    BEntry::Vacant(entry) => {
+                        entry.insert(mf);
+                    }
+                    BEntry::Occupied(mut entry) => {
+                        let existent_mf = entry.get_mut();
+                        let existent_metrics = existent_mf.mut_metric();
+
+                        for metric in mf.take_metric().into_iter() {
+                            existent_metrics.push(metric);
+                        }
+                    }
+                }
+            }
+        }
+
+        // Now that MetricFamilies are all set, sort their Metrics
+        // lexicographically by their label values.
+        for mf in mf_by_name.values_mut() {
+            mf.mut_metric().sort_by(|m1, m2| {
+                let lps1 = m1.get_label();
+                let lps2 = m2.get_label();
+
+                if lps1.len() != lps2.len() {
+                    // This should not happen. The metrics are
+                    // inconsistent. However, we have to deal with the fact, as
+                    // people might use custom collectors or metric family injection
+                    // to create inconsistent metrics. So let's simply compare the
+                    // number of labels in this case. That will still yield
+                    // reproducible sorting.
+                    return lps1.len().cmp(&lps2.len());
+                }
+
+                for (lp1, lp2) in lps1.iter().zip(lps2.iter()) {
+                    if lp1.get_value() != lp2.get_value() {
+                        return lp1.get_value().cmp(lp2.get_value());
+                    }
+                }
+
+                // We should never arrive here. Multiple metrics with the same
+                // label set in the same scrape will lead to undefined ingestion
+                // behavior. However, as above, we have to provide stable sorting
+                // here, even for inconsistent metrics. So sort equal metrics
+                // by their timestamp, with missing timestamps (implying "now")
+                // coming last.
+                m1.get_timestamp_ms().cmp(&m2.get_timestamp_ms())
+            });
+        }
+
+        // Write out MetricFamilies sorted by their name.
+        mf_by_name
+            .into_values()
+            .map(|mut m| {
+                // Add registry common labels, if any.
+                if let Some(ref hmap) = self.labels {
+                    let pairs: Vec<proto::LabelPair> = hmap
+                        .iter()
+                        .map(|(k, v)| {
+                            let mut label = proto::LabelPair::default();
+                            label.set_name(k.to_string());
+                            label.set_value(v.to_string());
+                            label
+                        })
+                        .collect();
+
+                    for metric in m.mut_metric().iter_mut() {
+                        let mut labels: Vec<_> = metric.take_label().into();
+                        labels.append(&mut pairs.clone());
+                        metric.set_label(labels.into());
+                    }
+                }
+                m
+            })
+            .collect()
+    }
+}
+
+thread_local! {
+    static DEFAULT_REGISTRY: RefCell<Registry> = RefCell::new(Registry::default());
+}
+
+/// Registers a new [`Collector`] to be included in metrics collection. It
+/// returns an error if the descriptors provided by the [`Collector`] are invalid or
+/// if they - in combination with descriptors of already registered Collectors -
+/// do not fulfill the consistency and uniqueness criteria described in the
+/// [`Desc`](crate::core::Desc) documentation.
+pub fn register(c: Box<dyn Collector>) -> LuaResult<()> {
+    DEFAULT_REGISTRY.with_borrow_mut(|r| r.register(c))
+}
+
+/// Unregisters the [`Collector`] that equals the [`Collector`] passed in as
+/// an argument. (Two Collectors are considered equal if their Describe method
+/// yields the same set of descriptors.) The function returns an error if a
+/// [`Collector`] was not registered.
+pub fn unregister(c: Box<dyn Collector>) -> LuaResult<()> {
+    DEFAULT_REGISTRY.with_borrow_mut(|r| r.unregister(c))
+}
+
+/// Return all `MetricFamily` of `DEFAULT_REGISTRY`.
+pub fn gather() -> Vec<proto::MetricFamily> {
+    DEFAULT_REGISTRY.with_borrow(|r| r.gather())
+}
+
+pub fn set_labels(labels: HashMap<String, String>) {
+    DEFAULT_REGISTRY.with_borrow_mut(|r| r.set_labels(labels))
+}

metrics-scm-1.rockspec

@@ -22,6 +22,7 @@ build = {
     type = 'cmake',
     variables = {
         TARANTOOL_INSTALL_LUADIR = '$(LUADIR)',
+        TARANTOOL_INSTALL_LIBDIR = '$(LIBDIR)',
     },
 }
 

metrics/api.lua

@@ -8,6 +8,9 @@ local Counter = require('metrics.collectors.counter')
 local Gauge = require('metrics.collectors.gauge')
 local Histogram = require('metrics.collectors.histogram')
 local Summary = require('metrics.collectors.summary')
+local HistogramVec = require('metrics.collectors.histogram_vec')
+
+local rs = require('metrics.rs')
 
 local registry = rawget(_G, '__metrics_registry')
 if not registry then
@@ -86,6 +89,15 @@ local function histogram(name, help, buckets, metainfo)
     return registry:find_or_create(Histogram, name, help, buckets, metainfo)
 end
 
+local function histogram_vec(name, help, label_names, buckets, metainfo)
+    checks('string', 'string', '?table', '?table', '?table')
+    if buckets ~= nil and not Histogram.check_buckets(buckets) then
+        error('Invalid value for buckets')
+    end
+
+    return registry:find_or_create(HistogramVec, name, help, label_names, buckets, metainfo)
+end
+
 local function summary(name, help, objectives, params, metainfo)
     checks('string', '?string', '?table', {
         age_buckets_count = '?number',
@@ -123,6 +135,11 @@ local function set_global_labels(label_pairs)
     end
 
     registry:set_labels(label_pairs)
+    rs.set_labels(label_pairs)
+end
+
+local function get_global_labels()
+    return registry:get_labels()
 end
 
 return {
@@ -132,6 +149,7 @@ return {
     counter = counter,
     gauge = gauge,
     histogram = histogram,
+    histogram_vec = histogram_vec,
     summary = summary,
 
     collect = collect,
@@ -140,4 +158,5 @@ return {
     unregister_callback = unregister_callback,
     invoke_callbacks = invoke_callbacks,
     set_global_labels = set_global_labels,
+    get_global_labels = get_global_labels,
 }

metrics/collectors/histogram_vec.lua

@@ -0,0 +1,68 @@
+local Shared = require('metrics.collectors.shared')
+local HistogramVec = Shared:new_class('histogram', {})
+local rust = require('metrics.rs')
+
+---@param label_names string[]
+function HistogramVec:new(name, help, label_names, buckets, metainfo)
+    metainfo = table.copy(metainfo) or {}
+    local obj = Shared.new(self, name, help, metainfo)
+
+    obj._label_names = label_names or {}
+    obj.inner = rust.new_histogram_vec({
+        name = name,
+        help = help,
+        buckets = buckets, -- can be nil
+    }, obj._label_names)
+
+    return obj
+end
+
+local function to_values(label_pairs, keys)
+    local n = #keys
+    local values = table.new(n, 0)
+    for i, name in ipairs(keys) do
+        values[i] = label_pairs[name]
+    end
+    return values
+end
+
+function HistogramVec:observe(value, label_pairs)
+    if value ~= nil then
+        value = tonumber(value)
+        if type(value) ~= 'number' then
+            error("Histogram observation should be a number")
+        end
+    end
+
+    local label_values
+    if type(label_pairs) == 'table' then
+        label_values = to_values(label_pairs, self._label_names)
+    end
+    self.inner:observe(value, label_values)
+end
+
+function HistogramVec:remove(label_pairs)
+    assert(label_pairs, 'label pairs is a required parameter')
+    local label_values = to_values(label_pairs, self._label_names)
+    self.inner:remove(label_values)
+end
+
+-- Fast collect
+function HistogramVec:collect_str()
+    return self.inner:collect_str()
+end
+
+-- Slow collect
+function HistogramVec:collect()
+    return self.inner:collect()
+end
+
+function HistogramVec:unregister()
+    if self.registry then
+        self.registry:unregister(self)
+    end
+    self.inner:unregister()
+    self.inner = nil
+end
+
+return HistogramVec

metrics/init.lua

@@ -16,6 +16,7 @@ return setmetatable({
     counter = api.counter,
     gauge = api.gauge,
     histogram = api.histogram,
+    histogram_vec = api.histogram_vec,
     summary = api.summary,
 
     INF = const.INF,
@@ -27,6 +28,7 @@ return setmetatable({
     unregister_callback = api.unregister_callback,
     invoke_callbacks = api.invoke_callbacks,
     set_global_labels = api.set_global_labels,
+    get_global_labels = api.get_global_labels,
     enable_default_metrics = tarantool.enable,
     cfg = cfg.cfg,
     http_middleware = http_middleware,

metrics/plugins/prometheus.lua

@@ -1,6 +1,8 @@
 local metrics = require('metrics')
 require('checks')
 
+local rs = require('metrics.rs')
+
 local prometheus = {}
 
 local function escape(str)
@@ -52,17 +54,23 @@ local function collect_and_serialize()
     metrics.invoke_callbacks()
     local parts = {}
     for _, c in pairs(metrics.collectors()) do
-        table.insert(parts, string.format("# HELP %s %s", c.name, c.help))
-        table.insert(parts, string.format("# TYPE %s %s", c.name, c.kind))
-        for _, obs in ipairs(c:collect()) do
-            local s = string.format('%s%s %s',
-                serialize_name(obs.metric_name),
-                serialize_label_pairs(obs.label_pairs),
-                serialize_value(obs.value)
-            )
-            table.insert(parts, s)
+        if not c.collect_str then
+            table.insert(parts, string.format("# HELP %s %s", c.name, c.help))
+            table.insert(parts, string.format("# TYPE %s %s", c.name, c.kind))
+            for _, obs in ipairs(c:collect()) do
+                local s = string.format('%s%s %s',
+                    serialize_name(obs.metric_name),
+                    serialize_label_pairs(obs.label_pairs),
+                    serialize_value(obs.value)
+                )
+                table.insert(parts, s)
+            end
         end
     end
+    local text = rs.gather(metrics.get_global_labels())
+    if text ~=  "" then
+        table.insert(parts, text)
+    end
     return table.concat(parts, '\n') .. '\n'
 end
 

metrics/registry.lua

@@ -12,6 +12,7 @@ function Registry:clear()
     self.collectors = {}
     self.callbacks = {}
     self.label_pairs = {}
+    collectgarbage('collect')
 end
 
 function Registry:find(kind, name)
@@ -40,6 +41,7 @@ end
 
 function Registry:unregister(collector)
     self.collectors[collector.name .. collector.kind] = nil
+    collectgarbage('collect')
 end
 
 function Registry:invoke_callbacks()
@@ -70,4 +72,8 @@ function Registry:set_labels(label_pairs)
     self.label_pairs = table.copy(label_pairs)
 end
 
+function Registry:get_labels()
+    return table.copy(self.label_pairs)
+end
+
 return Registry

metrics/rs.lua

@@ -0,0 +1,25 @@
+--- Bridge between Lua and Rust
+--- We do some conversions here, for speed-ups
+local M = {}
+local checks = require('checks')
+local rust = require('metrics_rs')
+
+function M.set_labels(label_pairs)
+    checks('table')
+    label_pairs = table.copy(label_pairs)
+    for k, v in pairs(label_pairs) do
+        label_pairs[k] = tostring(v)
+    end
+    rust.set_labels(label_pairs)
+end
+
+function M.gather()
+    return rust.gather()
+end
+
+function M.new_histogram_vec(opts, label_names)
+    checks('table', '?table')
+    return rust.new_histogram_vec(opts, label_names)
+end
+
+return M

test/collectors/histogram_vec_test.lua

@@ -0,0 +1,116 @@
+local t = require('luatest')
+local g = t.group('histogram_vec')
+
+local metrics = require('metrics')
+local HistogramVec = require('metrics.collectors.histogram_vec')
+local utils = require('test.utils')
+
+g.before_all(utils.create_server)
+g.after_all(utils.drop_server)
+
+g.before_each(metrics.clear)
+
+g.test_unsorted_buckets_error = function()
+    t.assert_error_msg_contains('Invalid value for buckets', metrics.histogram_vec, 'latency', 'help', nil, {0.9, 0.5})
+end
+
+g.test_remove_metric_by_label = function()
+    local h = HistogramVec:new('hist', 'some histogram', {'label'}, {2, 4})
+
+    h:observe(3, {label = '1'})
+    h:observe(5, {label = '2'})
+
+    utils.assert_observations(h:collect(),
+        {
+            {'hist_count', 1, {label = '1'}},
+            {'hist_sum', 3, {label = '1'}},
+            {'hist_bucket', 0, {label = '1', le = 2}},
+            {'hist_bucket', 1, {label = '1', le = 4}},
+            {'hist_bucket', 1, {label = '1', le = '+Inf'}},
+            {'hist_count', 1, {label = '2'}},
+            {'hist_sum', 5, {label = '2'}},
+            {'hist_bucket', 0, {label = '2', le = 4}},
+            {'hist_bucket', 0, {label = '2', le = 2}},
+            {'hist_bucket', 1, {label = '2', le = '+Inf'}},
+        }
+    )
+
+    h:remove({label = '1'})
+    utils.assert_observations(h:collect(),
+        {
+            {'hist_count', 1, {label = '2'}},
+            {'hist_sum', 5, {label = '2'}},
+            {'hist_bucket', 0, {label = '2', le = 2}},
+            {'hist_bucket', 0, {label = '2', le = 4}},
+            {'hist_bucket', 1, {label = '2', le = '+Inf'}},
+        }
+    )
+end
+
+g.test_histogram = function()
+    t.assert_error_msg_contains("bad argument #1 to histogram_vec (string expected, got nil)", function()
+        metrics.histogram_vec()
+    end)
+
+    t.assert_error_msg_contains("bad argument #1 to histogram_vec (string expected, got number)", function()
+        metrics.histogram_vec(2)
+    end)
+
+    local h = metrics.histogram_vec('hist', 'some histogram', {'foo'}, {2, 4})
+
+    h:observe(3, {foo = ''})
+    h:observe(5, {foo = ''})
+
+    local collectors = metrics.collectors()
+    t.assert_equals(utils.len(collectors), 1, 'histogram seen as only 1 collector')
+
+    local observations = metrics.collect()
+    local obs_sum = utils.find_obs('hist_sum', {foo=''}, observations)
+    local obs_count = utils.find_obs('hist_count', {foo=''}, observations)
+    local obs_bucket_2 = utils.find_obs('hist_bucket', { le = 2, foo='' }, observations)
+    local obs_bucket_4 = utils.find_obs('hist_bucket', { le = 4, foo='' }, observations)
+    local obs_bucket_inf = utils.find_obs('hist_bucket', { le = '+Inf', foo='' }, observations)
+
+    t.assert_equals(#observations, 5, '<name>_sum, <name>_count, and <name>_bucket with 3 labelpairs')
+    t.assert_equals(obs_sum.value, 8, '3 + 5 = 8')
+    t.assert_equals(obs_count.value, 2, '2 observed values')
+    t.assert_equals(obs_bucket_2.value, 0, 'bucket 2 has no values')
+    t.assert_equals(obs_bucket_4.value, 1, 'bucket 4 has 1 value: 3')
+    t.assert_equals(obs_bucket_inf.value, 2, 'bucket +inf has 2 values: 3, 5')
+
+    h:observe(3, { foo = 'bar' })
+
+    collectors = metrics.collectors()
+    t.assert_equals(utils.len(collectors), 1, 'still histogram seen as only 1 collector')
+    observations = metrics.collect()
+    obs_sum = utils.find_obs('hist_sum', { foo = 'bar' }, observations)
+    obs_count = utils.find_obs('hist_count', { foo = 'bar' }, observations)
+    obs_bucket_2 = utils.find_obs('hist_bucket', { le = 2, foo = 'bar' }, observations)
+    obs_bucket_4 = utils.find_obs('hist_bucket', { le = 4, foo = 'bar' }, observations)
+    obs_bucket_inf = utils.find_obs('hist_bucket', { le = '+Inf', foo = 'bar' }, observations)
+
+    t.assert_equals(#observations, 10, '+ <name>_sum, <name>_count, and <name>_bucket with 3 labelpairs')
+    t.assert_equals(obs_sum.value, 3, '3 = 3')
+    t.assert_equals(obs_count.value, 1, '1 observed values')
+    t.assert_equals(obs_bucket_2.value, 0, 'bucket 2 has no values')
+    t.assert_equals(obs_bucket_4.value, 1, 'bucket 4 has 1 value: 3')
+    t.assert_equals(obs_bucket_inf.value, 1, 'bucket +inf has 1 value: 3')
+end
+
+g.test_insert_non_number = function()
+    local h = metrics.histogram_vec('hist', 'some histogram', nil, {2, 4})
+    t.assert_error_msg_contains('Histogram observation should be a number', h.observe, h, true)
+end
+
+g.test_metainfo = function()
+    local metainfo = {my_useful_info = 'here'}
+    local h = metrics.histogram_vec('hist', 'some histogram', nil, {2, 4}, metainfo)
+    t.assert_equals(h.metainfo, metainfo)
+end
+
+g.test_metainfo_immutable = function()
+    local metainfo = {my_useful_info = 'here'}
+    local h = metrics.histogram_vec('hist', 'some histogram', nil, {2, 4}, metainfo)
+    metainfo['my_useful_info'] = 'there'
+    t.assert_equals(h.metainfo, {my_useful_info = 'here'})
+end

test/helper.lua

@@ -14,6 +14,9 @@ package.loaded['test.utils'].LUA_PATH = root .. '/?.lua;' ..
     root .. '/.rocks/share/tarantool/?.lua;' ..
     root .. '/.rocks/share/tarantool/?/init.lua'
 
+local ext = jit.os == 'OSX' and 'dylib' or 'so'
+package.loaded['test.utils'].LUA_CPATH = root .. '/.rocks/lib/tarantool/?.'..ext
+
 local t = require('luatest')
 local ok, cartridge_helpers = pcall(require, 'cartridge.test-helpers')
 if not ok then

test/tarantool3_helpers/server.lua

@@ -185,6 +185,9 @@ function Server:initialize()
     if self.env['LUA_PATH'] == nil then
         self.env['LUA_PATH'] = utils.LUA_PATH
     end
+    if self.env['LUA_CPATH'] == nil then
+        self.env['LUA_CPATH'] = utils.LUA_CPATH
+    end
 
     getmetatable(getmetatable(self)).initialize(self)
 end

test/utils.lua

@@ -11,7 +11,8 @@ function utils.create_server(g)
     g.server = t.Server:new({
         alias = 'myserver',
         env = {
-            LUA_PATH = utils.LUA_PATH
+            LUA_PATH = utils.LUA_PATH,
+            LUA_CPATH = utils.LUA_CPATH,
         }
     })
     g.server:start{wait_until_ready = true}
@@ -116,7 +117,8 @@ function utils.is_tarantool_3_config_supported()
 end
 
 -- Empty by default. Empty LUA_PATH satisfies built-in package tests.
--- For tarantool/metrics, LUA_PATH is set up through test.helper
+-- For tarantool/metrics, LUA_PATH, LUA_CPATH are set up through test.helper
 utils.LUA_PATH = nil
+utils.LUA_CPATH = nil
 
 return utils