Changeset

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_storage_metronome_readonly/README.markdown	Fri Jan 10 23:17:09 2025 +0100
@@ -0,0 +1,48 @@
+---
+labels:
+- 'Type-Storage'
+- 'Stage-Alpha'
+summary: Metronome Read-only Storage Module
+...
+
+Introduction
+============
+
+This is a storage backend using Metronome Lua storage.
+
+This module only works in read-only, and was made to be used by [mod\_migrate]
+to migrate from Metronome’s storage.
+
+So far it has only been tested migrating to sqlite, because
+mod\_storage\_internal relies on the same `data_path` variable as this module,
+and thus would overwrite the files we just read.
+
+I’ve also only tested it on a dump from a Metronome configured by Yunohost, so
+using LDAP and such for user accounts, I don’t yet know how to migrate from
+different Metronome account storages.
+
+Configuration
+=============
+
+Copy the module to the prosody modules/plugins directory.
+
+In Prosody's configuration file, set:
+
+    storage = "metronome_readonly"
+    data_path = "/var/lib/metronome"
+
+To run the actual migration, run this command:
+
+    ./prosodyctl mod_migrate kl.netlib.re roster,vcard,private,cloud_notify,pep,pep-archive,offline-archive,archive-archive sql
+
+It will create a file in `/var/lib/metronome/prosody.sqlite`, after which you
+can change your configuration file to point to it, or alternatively you can
+perform a second migration to the internal storage if you prefer that.
+
+Compatibility
+=============
+
+  ------------------------ --------
+  trunk (as of 2025-01-10) Works
+  0.12                     Untested
+  ------------------------ --------

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_storage_metronome_readonly/mod_storage_metronome_readonly.lua	Fri Jan 10 23:17:09 2025 +0100
@@ -0,0 +1,332 @@
+local datamanager = require "prosody.core.storagemanager".olddm;
+local datetime = require "prosody.util.datetime";
+local st = require "prosody.util.stanza";
+local now = require "prosody.util.time".now;
+local id = require "prosody.util.id".medium;
+local set = require "prosody.util.set";
+local envloadfile = require"prosody.util.envload".envloadfile;
+
+local host = module.host;
+
+local archive_item_limit = module:get_option_integer("storage_archive_item_limit", 10000, 0);
+
+-- Metronome doesn’t store the item publish time, so fallback to the migration time.
+local when = math.floor(now());
+
+local function encode (s)
+        return s and (s:gsub("%W", function (c) return string.format("%%%02x", c:byte()); end));
+end
+
+local driver = {};
+
+function driver:open(store, typ)
+	local mt = self[typ or "keyval"]
+	if not mt then
+		return nil, "unsupported-store";
+	end
+	return setmetatable({ store = store, type = typ }, mt);
+end
+
+function driver:stores(username) -- luacheck: ignore 212/self
+	if username == true then
+		local nodes = set.new();
+		for user in datamanager.users(host, "pep") do
+			local data = datamanager.load(user, host, "pep");
+
+			for _, node in ipairs(data["nodes"]) do
+				nodes:add("pep_" .. node);
+			end
+		end
+		return function()
+			for node in nodes do
+				nodes:remove(node);
+				return node;
+			end
+		end;
+	end
+end
+
+function driver:purge(user) -- luacheck: ignore 212/self
+	return nil, "unsupported-store";
+end
+
+local keyval = { };
+driver.keyval = { __index = keyval };
+
+function keyval:get(user)
+	if self.store == "pep" then
+		local data = datamanager.load(user, host, self.store);
+		local nodes = data["nodes"];
+		local result = {};
+
+		local pep_base_path = datamanager.getpath(user, host, self.store):sub(1, -5);
+
+		for _, node in ipairs(nodes) do
+			local path = ("%s/%s.dat"):format(pep_base_path, encode(node));
+			local data = envloadfile(path, {});
+			if not data then
+				log("error", "Failed to load metronome storage");
+				return nil, "Error reading storage";
+			end
+			local success, data = pcall(data);
+			if not success then
+				log("error", "Unable to load metronome storage");
+				return nil, "Error reading storage";
+			end
+			local new_node = {};
+			new_node["name"] = node;
+			new_node["subscribers"] = data["subscribers"];
+			new_node["affiliations"] = data["affiliations"];
+			new_node["config"] = data["config"];
+			result[node] = new_node;
+		end
+		return result;
+	elseif self.store == "cloud_notify" then
+		local data = datamanager.load(user, host, "push");
+		local result = {};
+		for jid, data in pairs(data) do
+			local secret = data["secret"];
+			for node in pairs(data["nodes"]) do
+				-- TODO: Does Metronome store more info than that?
+				local options;
+				if secret then
+					options = st.preserialize(st.stanza("x", { xmlns = "jabber:x:data", type = "submit" })
+						:tag("field", { var = "FORM_TYPE" })
+							:text_tag("value", "http://jabber.org/protocol/pubsub#publish-options")
+						:up()
+						:tag("field", { var = "secret" })
+							:text_tag("value", secret));
+				end
+				result[jid.."<"..node] = {
+					jid = jid,
+					node = node,
+					options = options,
+				};
+			end
+		end
+		return result;
+	elseif self.store == "roster" then
+		return datamanager.load(user, host, self.store);
+	elseif self.store == "vcard" then
+		return datamanager.load(user, host, self.store);
+	elseif self.store == "private" then
+		return datamanager.load(user, host, self.store);
+
+	-- After that, handle MUC specific stuff, not tested yet whatsoever.
+	elseif self.store == "persistent" then
+		return datamanager.load(user, host, self.store);
+	elseif self.store == "config" then
+		return datamanager.load(user, host, self.store);
+	elseif self.store == "vcard_muc" then
+		local data = datamanager.load(user, host, "room_icons");
+		return data and data["photo"];
+	else
+		return nil, "unsupported-store";
+	end
+end
+
+function keyval:set(user, data)
+	return nil, "unsupported-store";
+end
+
+function keyval:users()
+	local store;
+	if self.store == "vcard_muc" then
+		store = "room_icons";
+	elseif self.store == "cloud_notify" then
+		store = "push";
+	else
+		store = self.store;
+	end
+	return datamanager.users(host, store, self.type);
+end
+
+local archive = {};
+driver.archive = { __index = archive };
+
+archive.caps = {
+	total = true;
+	quota = archive_item_limit;
+	full_id_range = true;
+	ids = true;
+};
+
+function archive:append(username, key, value, when, with)
+	return nil, "unsupported-store";
+end
+
+function archive:find(username, query)
+	if self.store == "archive" then
+		local jid = username.."@"..host;
+		local data = datamanager.load(username, host, "archiving");
+		local iter = ipairs(data["logs"]);
+		local i = 0;
+		local message;
+		return function()
+			i, message = iter(data["logs"], i);
+			if not message then
+				return;
+			end
+
+			local with;
+			local bare_to = message["bare_to"];
+			local bare_from = message["bare_from"];
+			if jid == bare_to then
+				-- received
+				with = bare_from;
+			else
+				-- sent
+				with = bare_to;
+			end
+
+			local to = message["to"];
+			local from = message["from"];
+			local id = message["id"];
+			local type = message["type"];
+
+			local key = message["uid"];
+			local when = message["timestamp"];
+			local item = st.message({ to = to, from = from, id = id, type = type }, message["body"]);
+			if message["tags"] then
+				for _, tag in ipairs(message["tags"]) do
+					setmetatable(tag, st.stanza_mt);
+					item:add_direct_child(tag);
+				end
+			end
+			if message["marker"] then
+				item:tag(message["marker"], { xmlns = "urn:xmpp:chat-markers:0", id = message["marker_id"] });
+			end
+			return key, item, when, with;
+		end;
+
+	elseif self.store:sub(1, 4) == "pep_" then
+		local node = self.store:sub(5);
+
+		local pep_base_path = datamanager.getpath(username, host, "pep"):sub(1, -5);
+		local path = ("%s/%s.dat"):format(pep_base_path, encode(node));
+		local data = envloadfile(path, {});
+		if not data then
+			log("debug", "Failed to load metronome storage");
+			return {};
+		end
+		local success, data = pcall(data);
+		if not success then
+			log("error", "Unable to load metronome storage");
+			return nil, "Error reading storage";
+		end
+
+		local iter = pairs(data["data"]);
+		local key = nil;
+		local payload;
+		return function()
+			key, payload = iter(data["data"], key);
+			if not key then
+				return;
+			end
+			local item = st.deserialize(payload[1]);
+			local with = data["data_author"][key];
+			return key, item, when, with;
+		end;
+	elseif self.store == "offline" then
+		-- This is mostly copy/pasted from mod_storage_internal.
+		local list, err = datamanager.list_open(username, host, self.store);
+		if not list then
+			if err then
+				return list, err;
+			end
+			return function()
+			end;
+		end
+
+		local i = 0;
+		local iter = function()
+			i = i + 1;
+			return list[i];
+		end
+
+		return function()
+			local item = iter();
+			print(item)
+			if item == nil then
+				if list.close then
+					list:close();
+				end
+				return
+			end
+			print(0)
+			local key = id();
+			local when = item.attr and datetime.parse(item.attr.stamp);
+			local with = "";
+			print(1)
+			item.key, item.when, item.with = nil, nil, nil;
+			item.attr.stamp = nil;
+			-- COMPAT Stored data may still contain legacy XEP-0091 timestamp
+			item.attr.stamp_legacy = nil;
+			item = st.deserialize(item);
+			print(key, when, with, item)
+			return key, item, when, with;
+		end
+	else
+		return nil, "unsupported-store";
+	end
+end
+
+function archive:get(username, wanted_key)
+	local iter, err = self:find(username, { key = wanted_key })
+	if not iter then return iter, err; end
+	for key, stanza, when, with in iter do
+		if key == wanted_key then
+			return stanza, when, with;
+		end
+	end
+	return nil, "item-not-found";
+end
+
+function archive:set(username, key, new_value, new_when, new_with)
+	return nil, "unsupported-store";
+end
+
+function archive:dates(username)
+	return nil, "unsupported-store";
+end
+
+function archive:summary(username, query)
+	return nil, "unsupported-store";
+end
+
+function archive:users()
+	if self.store == "archive" then
+		return datamanager.users(host, "archiving");
+	elseif self.store:sub(1, 4) == "pep_" then
+		local wanted_node = self.store:sub(5);
+		local iter, tbl = datamanager.users(host, "pep");
+		return function()
+			while true do
+				local user = iter(tbl);
+				if not user then
+					return;
+				end
+				local data = datamanager.load(user, host, "pep");
+				for _, node in ipairs(data["nodes"]) do
+					if node == wanted_node then
+						return user;
+					end
+				end
+			end
+		end;
+	elseif self.store == "offline" then
+		return datamanager.users(host, self.store, "list");
+	else
+		return nil, "unsupported-store";
+	end
+end
+
+function archive:trim(username, to_when)
+	return nil, "unsupported-store";
+end
+
+function archive:delete(username, query)
+	return nil, "unsupported-store";
+end
+
+module:provides("storage", driver);