File

core/configmanager.lua @ 11523:5f15ab7c6ae5

Statistics: Rewrite statistics backends to use OpenMetrics The metric subsystem of Prosody has had some shortcomings from the perspective of the current state-of-the-art in metric observability. The OpenMetrics standard [0] is a formalization of the data model (and serialization format) of the well-known and widely-used Prometheus [1] software stack. The previous stats subsystem of Prosody did not map well to that format (see e.g. [2] and [3]); the key reason is that it was trying to do too much math on its own ([2]) while lacking first-class support for "families" of metrics ([3]) and structured metric metadata (despite the `extra` argument to metrics, there was no standard way of representing common things like "tags" or "labels"). Even though OpenMetrics has grown from the Prometheus world of monitoring, it maps well to other popular monitoring stacks such as: - InfluxDB (labels can be mapped to tags and fields as necessary) - Carbon/Graphite (labels can be attached to the metric name with dot-separation) - StatsD (see graphite when assuming that graphite is used as backend, which is the default) The util.statsd module has been ported to use the OpenMetrics model as a proof of concept. An implementation which exposes the util.statistics backend data as Prometheus metrics is ready for publishing in prosody-modules (most likely as mod_openmetrics_prometheus to avoid breaking existing 0.11 deployments). At the same time, the previous measure()-based API had one major advantage: It is really simple and easy to use without requiring lots of knowledge about OpenMetrics or similar concepts. For that reason as well as compatibility with existing code, it is preserved and may even be extended in the future. However, code relying on the `stats-updated` event as well as `get_stats` from `statsmanager` will break because the data model has changed completely; in case of `stats-updated`, the code will simply not run (as the event was renamed in order to avoid conflicts); the `get_stats` function has been removed completely (so it will cause a traceback when it is attempted to be used). Note that the measure_*_event methods have been removed from the module API. I was unable to find any uses or documentation and thus deemed they should not be ported. Re-implementation is possible when necessary. [0]: https://openmetrics.io/ [1]: https://prometheus.io/ [2]: #959 [3]: #960
author Jonas Schäfer <jonas@wielicki.name>
date Sun, 18 Apr 2021 11:47:41 +0200
parent 10375:3d0adbc74c39
child 12083:ec21e379c145
line wrap: on
line source

-- Prosody IM
-- Copyright (C) 2008-2010 Matthew Wild
-- Copyright (C) 2008-2010 Waqas Hussain
--
-- This project is MIT/X11 licensed. Please see the
-- COPYING file in the source package for more information.
--

local _G = _G;
local setmetatable, rawget, rawset, io, os, error, dofile, type, pairs, ipairs =
      setmetatable, rawget, rawset, io, os, error, dofile, type, pairs, ipairs;
local format, math_max, t_insert = string.format, math.max, table.insert;

local envload = require"util.envload".envload;
local deps = require"util.dependencies";
local resolve_relative_path = require"util.paths".resolve_relative_path;
local glob_to_pattern = require"util.paths".glob_to_pattern;
local path_sep = package.config:sub(1,1);
local get_traceback_table = require "util.debug".get_traceback_table;

local encodings = deps.softreq"util.encodings";
local nameprep = encodings and encodings.stringprep.nameprep or function (host) return host:lower(); end

local _M = {};
local _ENV = nil;
-- luacheck: std none

_M.resolve_relative_path = resolve_relative_path; -- COMPAT

local parser = nil;

local config_mt = { __index = function (t, _) return rawget(t, "*"); end};
local config = setmetatable({ ["*"] = { } }, config_mt);

-- When host not found, use global
local host_mt = { __index = function(_, k) return config["*"][k] end }

function _M.getconfig()
	return config;
end

function _M.get(host, key, _oldkey)
	if key == "core" then
		key = _oldkey; -- COMPAT with code that still uses "core"
	end
	return config[host][key];
end
function _M.rawget(host, key, _oldkey)
	if key == "core" then
		key = _oldkey; -- COMPAT with code that still uses "core"
	end
	local hostconfig = rawget(config, host);
	if hostconfig then
		return rawget(hostconfig, key);
	end
end

local function set(config_table, host, key, value)
	if host and key then
		local hostconfig = rawget(config_table, host);
		if not hostconfig then
			hostconfig = rawset(config_table, host, setmetatable({}, host_mt))[host];
		end
		hostconfig[key] = value;
		return true;
	end
	return false;
end

function _M.set(host, key, value, _oldvalue)
	if key == "core" then
		key, value = value, _oldvalue; --COMPAT with code that still uses "core"
	end
	return set(config, host, key, value);
end

function _M.load(filename, config_format)
	config_format = config_format or filename:match("%w+$");

	if config_format == "lua" then
		local f, err = io.open(filename);
		if f then
			local new_config = setmetatable({ ["*"] = { } }, config_mt);
			local ok, err = parser.load(f:read("*a"), filename, new_config);
			f:close();
			if ok then
				config = new_config;
			end
			return ok, "parser", err;
		end
		return f, "file", err;
	end

	if not config_format then
		return nil, "file", "no parser specified";
	else
		return nil, "file", "no parser for "..(config_format);
	end
end

-- Built-in Lua parser
do
	local pcall = _G.pcall;
	local function get_line_number(config_file)
		local tb = get_traceback_table(nil, 2);
		for i = 1, #tb do
			if tb[i].info.short_src == config_file then
				return tb[i].info.currentline;
			end
		end
	end
	parser = {};
	function parser.load(data, config_file, config_table)
		local set_options = {}; -- set_options[host.."/"..option_name] = true (when the option has been set already in this file)
		local warnings = {};
		local env;
		-- The ' = true' are needed so as not to set off __newindex when we assign the functions below
		env = setmetatable({
			Host = true, host = true, VirtualHost = true,
			Component = true, component = true,
			Include = true, include = true, RunScript = true }, {
				__index = function (_, k)
					if k:match("^ENV_") then
						return os.getenv(k:sub(5));
					end
					return rawget(_G, k);
				end,
				__newindex = function (_, k, v)
					local host = env.__currenthost or "*";
					local option_path = host.."/"..k;
					if set_options[option_path] then
						t_insert(warnings, ("%s:%d: Duplicate option '%s'"):format(config_file, get_line_number(config_file), k));
					end
					set_options[option_path] = true;
					set(config_table, env.__currenthost or "*", k, v);
				end
		});

		rawset(env, "__currenthost", "*") -- Default is global
		function env.VirtualHost(name)
			if not name then
				error("Host must have a name", 2);
			end
			local prepped_name = nameprep(name);
			if not prepped_name then
				error(format("Name of Host %q contains forbidden characters", name), 0);
			end
			name = prepped_name;
			if rawget(config_table, name) and rawget(config_table[name], "component_module") then
				error(format("Host %q clashes with previously defined %s Component %q, for services use a sub-domain like conference.%s",
					name, config_table[name].component_module:gsub("^%a+$", { component = "external", muc = "MUC"}), name, name), 0);
			end
			rawset(env, "__currenthost", name);
			-- Needs at least one setting to logically exist :)
			set(config_table, name or "*", "defined", true);
			return function (config_options)
				rawset(env, "__currenthost", "*"); -- Return to global scope
				for option_name, option_value in pairs(config_options) do
					set(config_table, name or "*", option_name, option_value);
				end
			end;
		end
		env.Host, env.host = env.VirtualHost, env.VirtualHost;

		function env.Component(name)
			if not name then
				error("Component must have a name", 2);
			end
			local prepped_name = nameprep(name);
			if not prepped_name then
				error(format("Name of Component %q contains forbidden characters", name), 0);
			end
			name = prepped_name;
			if rawget(config_table, name) and rawget(config_table[name], "defined")
				and not rawget(config_table[name], "component_module") then
				error(format("Component %q clashes with previously defined Host %q, for services use a sub-domain like conference.%s",
					name, name, name), 0);
			end
			set(config_table, name, "component_module", "component");
			-- Don't load the global modules by default
			set(config_table, name, "load_global_modules", false);
			rawset(env, "__currenthost", name);
			local function handle_config_options(config_options)
				rawset(env, "__currenthost", "*"); -- Return to global scope
				for option_name, option_value in pairs(config_options) do
					set(config_table, name or "*", option_name, option_value);
				end
			end

			return function (module)
					if type(module) == "string" then
						set(config_table, name, "component_module", module);
						return handle_config_options;
					end
					return handle_config_options(module);
				end
		end
		env.component = env.Component;

		function env.Include(file)
			-- Check whether this is a wildcard Include
			if file:match("[*?]") then
				local lfs = deps.softreq "lfs";
				if not lfs then
					error(format("Error expanding wildcard pattern in Include %q - LuaFileSystem not available", file));
				end
				local path_pos, glob = file:match("()([^"..path_sep.."]+)$");
				local path = file:sub(1, math_max(path_pos-2,0));
				local config_path = config_file:gsub("[^"..path_sep.."]+$", "");
				if #path > 0 then
					path = resolve_relative_path(config_path, path);
				else
					path = config_path;
				end
				local patt = glob_to_pattern(glob);
				for f in lfs.dir(path) do
					if f:sub(1,1) ~= "." and f:match(patt) then
						env.Include(path..path_sep..f);
					end
				end
				return;
			end
			-- Not a wildcard, so resolve (potentially) relative path and run through config parser
			file = resolve_relative_path(config_file:gsub("[^"..path_sep.."]+$", ""), file);
			local f, err = io.open(file);
			if f then
				local ret, err = parser.load(f:read("*a"), file, config_table);
				if not ret then error(err:gsub("%[string.-%]", file), 0); end
				if err then
					for _, warning in ipairs(err) do
						t_insert(warnings, warning);
					end
				end
			end
			if not f then error("Error loading included "..file..": "..err, 0); end
			return f, err;
		end
		env.include = env.Include;

		function env.RunScript(file)
			return dofile(resolve_relative_path(config_file:gsub("[^"..path_sep.."]+$", ""), file));
		end

		local chunk, err = envload(data, "@"..config_file, env);

		if not chunk then
			return nil, err;
		end

		local ok, err = pcall(chunk);

		if not ok then
			return nil, err;
		end

		return true, warnings;
	end

end

return _M;