Software `/` code `/` prosody

Comparison

plugins/mod_admin_shell.lua @ 11523:5f15ab7c6ae5

Statistics: Rewrite statistics backends to use OpenMetrics The metric subsystem of Prosody has had some shortcomings from the perspective of the current state-of-the-art in metric observability. The OpenMetrics standard [0] is a formalization of the data model (and serialization format) of the well-known and widely-used Prometheus [1] software stack. The previous stats subsystem of Prosody did not map well to that format (see e.g. [2] and [3]); the key reason is that it was trying to do too much math on its own ([2]) while lacking first-class support for "families" of metrics ([3]) and structured metric metadata (despite the `extra` argument to metrics, there was no standard way of representing common things like "tags" or "labels"). Even though OpenMetrics has grown from the Prometheus world of monitoring, it maps well to other popular monitoring stacks such as: - InfluxDB (labels can be mapped to tags and fields as necessary) - Carbon/Graphite (labels can be attached to the metric name with dot-separation) - StatsD (see graphite when assuming that graphite is used as backend, which is the default) The util.statsd module has been ported to use the OpenMetrics model as a proof of concept. An implementation which exposes the util.statistics backend data as Prometheus metrics is ready for publishing in prosody-modules (most likely as mod_openmetrics_prometheus to avoid breaking existing 0.11 deployments). At the same time, the previous measure()-based API had one major advantage: It is really simple and easy to use without requiring lots of knowledge about OpenMetrics or similar concepts. For that reason as well as compatibility with existing code, it is preserved and may even be extended in the future. However, code relying on the `stats-updated` event as well as `get_stats` from `statsmanager` will break because the data model has changed completely; in case of `stats-updated`, the code will simply not run (as the event was renamed in order to avoid conflicts); the `get_stats` function has been removed completely (so it will cause a traceback when it is attempted to be used). Note that the measure_*_event methods have been removed from the module API. I was unable to find any uses or documentation and thus deemed they should not be ported. Re-implementation is possible when necessary. [0]: https://openmetrics.io/ [1]: https://prometheus.io/ [2]: #959 [3]: #960

author	Jonas Schäfer <jonas@wielicki.name>
date	Sun, 18 Apr 2021 11:47:41 +0200
parent	11504:1f700f5f62cb
child	11601:9483728f890f

comparison

equal deleted inserted replaced

-:5bd38d9197e1
+:5f15ab7c6ae5
 local has_pposix, pposix = pcall(require, "util.pposix");
 local async = require "util.async";
 local serialization = require "util.serialization";
 local serialize_config = serialization.new ({ fatal = false, unquoted = true});
 local time = require "util.time";
+local t_insert = table.insert;
+local t_concat = table.concat;
 local format_number = require "util.human.units".format;
 local format_table = require "util.human.io".table;
 local commands = module:shared("commands")
 local short_units = {
 	seconds = "s",
 	bytes = "B",
 };
-local function format_stat(type, unit, value, ref_value)
-	ref_value = ref_value or value;
-	--do return tostring(value) end
-	if not unit then
-		if type == "duration" then
-			unit = "seconds"
-		elseif type == "size" then
-			unit = "bytes";
-		elseif type == "rate" then
-			unit = " events/sec"
-			if ref_value < 0.9 then
-				unit = "events/min"
-				value = value*60;
-				if ref_value < 0.6/60 then
-					unit = "events/h"
-					value = value*60;
-				end
-			end
-			return ("%.3g %s"):format(value, unit);
-		end
-	end
-	return format_number(value, short_units[unit] or unit or "", unit == "bytes" and 'b' or nil);
-end
 local stats_methods = {};
-function stats_methods:bounds(_lower, _upper)
-	for _, stat_info in ipairs(self) do
+function stats_methods:render_single_fancy_histogram_ex(print, prefix, metric_family, metric, cumulative)
-		local data = stat_info[4];
+	local creation_timestamp, sum, count
-		if data then
+	local buckets = {}
-			local lower = _lower or data.min;
+	local prev_bucket_count = 0
-			local upper = _upper or data.max;
+	for suffix, extra_labels, value in metric:iter_samples() do
-			local new_data = {
+		if suffix == "_created" then
-				min = lower;
+			creation_timestamp = value
-				max = upper;
+		elseif suffix == "_sum" then
-				samples = {};
+			sum = value
-				sample_count = 0;
+		elseif suffix == "_count" then
-				count = data.count;
+			count = value
-				units = data.units;
+		else
-			};
+			local bucket_threshold = extra_labels["le"]
-			local sum = 0;
+			local bucket_count
-			for _, v in ipairs(data.samples) do
+			if cumulative then
-				if v > upper then
+				bucket_count = value
-					break;
+			else
-				elseif v>=lower then
+				bucket_count = value - prev_bucket_count
-					table.insert(new_data.samples, v);
+				prev_bucket_count = value
-					sum = sum + v;
+			end
-				end
+			if bucket_threshold == "+Inf" then
-			end
+				t_insert(buckets, {threshold = 1/0, count = bucket_count})
-			new_data.sample_count = #new_data.samples;
+			elseif bucket_threshold ~= nil then
-			stat_info[4] = new_data;
+				t_insert(buckets, {threshold = tonumber(bucket_threshold), count = bucket_count})
-			stat_info[3] = sum/new_data.sample_count;
+			end
 		end
 	end
-	return self;
-end
+	if #buckets == 0 or not creation_timestamp or not sum or not count then
+		print("[no data or not a histogram]")
-function stats_methods:trim(lower, upper)
+		return false
-	upper = upper or (100-lower);
+	end
-	local statistics = require "util.statistics";
-	for _, stat_info in ipairs(self) do
+	local graph_width, graph_height, wscale = #buckets, 10, 1;
-		-- Strip outliers
+	if graph_width < 8 then
-		local data = stat_info[4];
+		wscale = 8
-		if data then
+	elseif graph_width < 16 then
-			local new_data = {
+		wscale = 4
-				min = statistics.get_percentile(data, lower);
+	elseif graph_width < 32 then
-				max = statistics.get_percentile(data, upper);
+		wscale = 2
-				samples = {};
+	end
-				sample_count = 0;
+	local eighth_chars = "   ▁▂▃▄▅▆▇█";
-				count = data.count;
-				units = data.units;
+	local max_bin_samples = 0
-			};
+	for _, bucket in ipairs(buckets) do
-			local sum = 0;
+		if bucket.count > max_bin_samples then
-			for _, v in ipairs(data.samples) do
+			max_bin_samples = bucket.count
-				if v > new_data.max then
+		end
-					break;
+	end
-				elseif v>=new_data.min then
-					table.insert(new_data.samples, v);
+	print("");
-					sum = sum + v;
+	print(prefix)
-				end
+	print(("_"):rep(graph_width*wscale).." "..max_bin_samples);
-			end
+	for row = graph_height, 1, -1 do
-			new_data.sample_count = #new_data.samples;
+		local row_chars = {};
-			stat_info[4] = new_data;
+		local min_eighths, max_eighths = 8, 0;
-			stat_info[3] = sum/new_data.sample_count;
+		for i = 1, #buckets do
-		end
+			local char_eighths = math.ceil(math.max(math.min((graph_height/(max_bin_samples/buckets[i].count))-(row-1), 1), 0)*8);
-	end
+			if char_eighths < min_eighths then
-	return self;
+				min_eighths = char_eighths;
-end
+			end
+			if char_eighths > max_eighths then
-function stats_methods:max(upper)
+				max_eighths = char_eighths;
-	return self:bounds(nil, upper);
+			end
-end
+			if char_eighths == 0 then
+				row_chars[i] = ("-"):rep(wscale);
-function stats_methods:min(lower)
+			else
-	return self:bounds(lower, nil);
+				local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
-end
+				row_chars[i] = char:rep(wscale);
+			end
-function stats_methods:summary()
+		end
-	local statistics = require "util.statistics";
+		print(table.concat(row_chars).."|- "..string.format("%.8g", math.ceil((max_bin_samples/graph_height)*(row-0.5))));
-	for _, stat_info in ipairs(self) do
+	end
-		local type, value, data = stat_info[2], stat_info[3], stat_info[4];
-		if data and data.samples then
+	local legend_pat = string.format("%%%d.%dg", wscale-1, wscale-1)
-			table.insert(stat_info.output, string.format("Count: %d (%d captured)",
+	local row = {}
-				data.count,
+	for i = 1, #buckets do
-				data.sample_count
+		local threshold = buckets[i].threshold
-			));
+		t_insert(row, legend_pat:format(threshold))
-			table.insert(stat_info.output, string.format("Min: %s  Mean: %s  Max: %s",
+	end
-				format_stat(type, data.units, data.min),
+	t_insert(row, " " .. metric_family.unit)
-				format_stat(type, data.units, value),
+	print(t_concat(row, "/"))
-				format_stat(type, data.units, data.max)
-			));
+	return true
-			table.insert(stat_info.output, string.format("Q1: %s  Median: %s  Q3: %s",
+end
-				format_stat(type, data.units, statistics.get_percentile(data, 25)),
-				format_stat(type, data.units, statistics.get_percentile(data, 50)),
+function stats_methods:render_single_fancy_histogram(print, prefix, metric_family, metric)
-				format_stat(type, data.units, statistics.get_percentile(data, 75))
+	return self:render_single_fancy_histogram_ex(print, prefix, metric_family, metric, false)
-			));
+end
-		end
-	end
+function stats_methods:render_single_fancy_histogram_cf(print, prefix, metric_family, metric)
-	return self;
+	-- cf = cumulative frequency
+	return self:render_single_fancy_histogram_ex(print, prefix, metric_family, metric, true)
 end
 function stats_methods:cfgraph()
 	for _, stat_info in ipairs(self) do
-		local name, type, value, data = unpack(stat_info, 1, 4); -- luacheck: ignore 211
+		local family_name, metric_family = unpack(stat_info, 1, 2)
 		local function print(s)
 			table.insert(stat_info.output, s);
 		end
-		if data and data.sample_count and data.sample_count > 0 then
+		if not self:render_family(print, family_name, metric_family, self.render_single_fancy_histogram_cf) then
-			local raw_histogram = require "util.statistics".get_histogram(data);
+			return self
-			local graph_width, graph_height = 50, 10;
-			local eighth_chars = "   ▁▂▃▄▅▆▇█";
-			local range = data.max - data.min;
-			if range > 0 then
-				local x_scaling = #raw_histogram/graph_width;
-				local histogram = {};
-				for i = 1, graph_width do
-					histogram[i] = math.max(raw_histogram[i*x_scaling-1] or 0, raw_histogram[i*x_scaling] or 0);
-				end
-				print("");
-				print(("_"):rep(52)..format_stat(type, data.units, data.max));
-				for row = graph_height, 1, -1 do
-					local row_chars = {};
-					local min_eighths, max_eighths = 8, 0;
-					for i = 1, #histogram do
-						local char_eighths = math.ceil(math.max(math.min((graph_height/(data.max/histogram[i]))-(row-1), 1), 0)*8);
-						if char_eighths < min_eighths then
-							min_eighths = char_eighths;
-						end
-						if char_eighths > max_eighths then
-							max_eighths = char_eighths;
-						end
-						if char_eighths == 0 then
-							row_chars[i] = "-";
-						else
-							local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
-							row_chars[i] = char;
-						end
-					end
-					print(table.concat(row_chars).."|-"..format_stat(type, data.units, data.max/(graph_height/(row-0.5))));
-				end
-				print(("\\    "):rep(11));
-				local x_labels = {};
-				for i = 1, 11 do
-					local s = ("%-4s"):format((i-1)*10);
-					if #s > 4 then
-						s = s:sub(1, 3).."…";
-					end
-					x_labels[i] = s;
-				end
-				print(" "..table.concat(x_labels, " "));
-				local units = "%";
-				local margin = math.floor((graph_width-#units)/2);
-				print((" "):rep(margin)..units);
-			else
-				print("[range too small to graph]");
-			end
-			print("");
 		end
 	end
 	return self;
 end
 function stats_methods:histogram()
 	for _, stat_info in ipairs(self) do
-		local name, type, value, data = unpack(stat_info, 1, 4); -- luacheck: ignore 211
+		local family_name, metric_family = unpack(stat_info, 1, 2)
 		local function print(s)
 			table.insert(stat_info.output, s);
 		end
-		if not data then
+		if not self:render_family(print, family_name, metric_family, self.render_single_fancy_histogram) then
-			print("[no data]");
+			return self
-			return self;
+		end
-		elseif not data.sample_count then
+	end
-			print("[not a sampled metric type]");
+	return self;
-			return self;
+end
-		end
+function stats_methods:render_single_counter(print, prefix, metric_family, metric)
-		local graph_width, graph_height = 50, 10;
+	local created_timestamp, current_value
-		local eighth_chars = "   ▁▂▃▄▅▆▇█";
+	for suffix, _, value in metric:iter_samples() do
+		if suffix == "_created" then
-		local range = data.max - data.min;
+			created_timestamp = value
+		elseif suffix == "_total" then
-		if range > 0 then
+			current_value = value
-			local n_buckets = graph_width;
+		end
+	end
-			local histogram = {};
+	if current_value and created_timestamp then
-			for i = 1, n_buckets do
+		local base_unit = short_units[metric_family.unit] or metric_family.unit
-				histogram[i] = 0;
+		local unit = base_unit .. "/s"
-			end
+		local factor = 1
-			local max_bin_samples = 0;
+		if base_unit == "s" then
-			for _, d in ipairs(data.samples) do
+			-- be smart!
-				local bucket = math.floor(1+(n_buckets-1)/(range/(d-data.min)));
+			unit = "%"
-				histogram[bucket] = histogram[bucket] + 1;
+			factor = 100
-				if histogram[bucket] > max_bin_samples then
+		elseif base_unit == "" then
-					max_bin_samples = histogram[bucket];
+			unit = "events/s"
-				end
+		end
-			end
+		print(("%-50s %s"):format(prefix, format_number(factor * current_value / (self.now - created_timestamp), unit.." [avg]")));
+	end
-			print("");
+end
-			print(("_"):rep(52)..max_bin_samples);
-			for row = graph_height, 1, -1 do
+function stats_methods:render_single_gauge(print, prefix, metric_family, metric)
-				local row_chars = {};
+	local current_value
-				local min_eighths, max_eighths = 8, 0;
+	for _, _, value in metric:iter_samples() do
-				for i = 1, #histogram do
+		current_value = value
-					local char_eighths = math.ceil(math.max(math.min((graph_height/(max_bin_samples/histogram[i]))-(row-1), 1), 0)*8);
+	end
-					if char_eighths < min_eighths then
+	if current_value then
-						min_eighths = char_eighths;
+		local unit = short_units[metric_family.unit] or metric_family.unit
-					end
+		print(("%-50s %s"):format(prefix, format_number(current_value, unit)));
-					if char_eighths > max_eighths then
+	end
-						max_eighths = char_eighths;
+end
-					end
-					if char_eighths == 0 then
+function stats_methods:render_single_summary(print, prefix, metric_family, metric)
-						row_chars[i] = "-";
+	local sum, count
-					else
+	for suffix, _, value in metric:iter_samples() do
-						local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
+		if suffix == "_sum" then
-						row_chars[i] = char;
+			sum = value
-					end
+		elseif suffix == "_count" then
-				end
+			count = value
-				print(table.concat(row_chars).."|-"..math.ceil((max_bin_samples/graph_height)*(row-0.5)));
+		end
-			end
+	end
-			print(("\\    "):rep(11));
+	if sum and count then
-			local x_labels = {};
+		local unit = short_units[metric_family.unit] or metric_family.unit
-			for i = 1, 11 do
+		if count == 0 then
-				local s = ("%-4s"):format(format_stat(type, data.units, data.min+range*i/11, data.min):match("^%S+"));
+			print(("%-50s %s"):format(prefix, "no obs."));
-				if #s > 4 then
-					s = s:sub(1, 3).."…";
-				end
-				x_labels[i] = s;
-			end
-			print(" "..table.concat(x_labels, " "));
-			local units = format_stat(type, data.units, data.min):match("%s+(.+)$") or data.units or "";
-			local margin = math.floor((graph_width-#units)/2);
-			print((" "):rep(margin)..units);
 		else
-			print("[range too small to graph]");
+			print(("%-50s %s"):format(prefix, format_number(sum / count, unit.."/event [avg]")));
 		end
-		print("");
+	end
-	end
+end
-	return self;
+function stats_methods:render_family(print, family_name, metric_family, render_func)
+	local labelkeys = metric_family.label_keys
+	if #labelkeys > 0 then
+		print(family_name)
+		for labelset, metric in metric_family:iter_metrics() do
+			local labels = {}
+			for i, k in ipairs(labelkeys) do
+				local v = labelset[i]
+				t_insert(labels, ("%s=%s"):format(k, v))
+			end
+			local prefix = "  "..t_concat(labels, " ")
+			render_func(self, print, prefix, metric_family, metric)
+		end
+	else
+		for _, metric in metric_family:iter_metrics() do
+			render_func(self, print, family_name, metric_family, metric)
+		end
+	end
 end
 local function stats_tostring(stats)
 	local print = stats.session.print;
 	for _, stat_info in ipairs(stats) do
 			for _, v in ipairs(stat_info.output) do
 				print(v);
 			end
 			print("");
 		else
-			print(("%-50s %s"):format(stat_info[1], format_stat(stat_info[2], (stat_info[4] or {}).units, stat_info[3])));
+			local metric_family = stat_info[2]
+			if metric_family.type_ == "counter" then
+				stats:render_family(print, stat_info[1], metric_family, stats.render_single_counter)
+			elseif metric_family.type_ == "gauge" or metric_family.type_ == "unknown" then
+				stats:render_family(print, stat_info[1], metric_family, stats.render_single_gauge)
+			elseif metric_family.type_ == "summary" or metric_family.type_ == "histogram" then
+				stats:render_family(print, stat_info[1], metric_family, stats.render_single_summary)
+			end
 		end
 	end
 	return #stats.." statistics displayed";
 end
 local stats_mt = {__index = stats_methods, __tostring = stats_tostring }
 local function new_stats_context(self)
-	return setmetatable({ session = self.session, stats = true }, stats_mt);
+	-- TODO: instead of now(), it might be better to take the time of the last
-end
+	-- interval, if the statistics backend is set to use periodic collection
+	-- Otherwise we get strange stuff like average cpu usage decreasing until
-function def_env.stats:show(filter)
+	-- the next sample and so on.
-	-- luacheck: ignore 211/changed
+	return setmetatable({ session = self.session, stats = true, now = time.now() }, stats_mt);
-	local stats, changed, extra = require "core.statsmanager".get_stats();
+end
-	local available, displayed = 0, 0;
+function def_env.stats:show(name_filter)
+	local statsman = require "core.statsmanager"
+	local collect = statsman.collect
+	if collect then
+		-- force collection if in manual mode
+		collect()
+	end
+	local metric_registry = statsman.get_metric_registry();
 	local displayed_stats = new_stats_context(self);
-	for name, value in iterators.sorted_pairs(stats) do
+	for family_name, metric_family in iterators.sorted_pairs(metric_registry:get_metric_families()) do
-		available = available + 1;
+		if not name_filter or family_name:match(name_filter) then
-		if not filter or name:match(filter) then
-			displayed = displayed + 1;
-			local type = name:match(":(%a+)$");
 			table.insert(displayed_stats, {
-				name, type, value, extra[name];
+				family_name,
-				output = {};
+				metric_family,
-			});
+				output = {}
+			})
 		end
 	end
 	return displayed_stats;
 end

prosody

5f15ab7c6ae5

plugins/mod_admin_shell.lua

Software / code / prosody

Comparison

plugins/mod_admin_shell.lua @ 11523:5f15ab7c6ae5

Software `/` code `/` prosody