prosody-modules
7f9bf161f640
mod_storage_xmlarchive/mod_storage_xmlarchive.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody-modules

Annotate

mod_storage_xmlarchive/mod_storage_xmlarchive.lua @ 2397:7f9bf161f640

mod_storage_xmlarchive: Open XML file later, just before it is needed
author Kim Alvefur <zash@zash.se>
date Fri, 25 Nov 2016 23:11:07 +0100
parent 2396:544ee057a23f
child 2398:7e922b968b44
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1764
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
1 -- mod_storage_xmlarchive
1765
1a7867eaac9e mod_storage_xmlarchive: Mistake in copypaste of copyright header :P
Kim Alvefur <zash@zash.se>
parents: 1764
diff changeset
2 -- Copyright (C) 2015 Kim Alvefur
1764
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
3 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
4 -- This file is MIT/X11 licensed.
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
5 --
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
6 -- luacheck: ignore unused self
a077dcf1bb36 mod_storage_xmlarchive: Add (c) header
Kim Alvefur <zash@zash.se>
parents: 1763
diff changeset
7
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
8 local lfs = require "lfs";
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 local dm = require "core.storagemanager".olddm;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10 local hmac_sha256 = require"util.hashes".hmac_sha256;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
11 local st = require"util.stanza";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
12 local dt = require"util.datetime";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
13 local new_stream = require "util.xmppstream".new;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
14 local empty = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
15
2289
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
16 if not dm.append_raw then
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
17 module:require"datamanager_append_raw";
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
18 end
aa984980a4dc mod_storage_xmlarchive: Include the missing append_raw() for 0.9 compatibility
Kim Alvefur <zash@zash.se>
parents: 2271
diff changeset
19
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
20 local archive = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
21 local archive_mt = { __index = archive };
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22
1753
54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc)
Kim Alvefur <zash@zash.se>
parents: 1752
diff changeset
23 function archive:append(username, _, data, when, with)
54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc)
Kim Alvefur <zash@zash.se>
parents: 1752
diff changeset
24 if type(when) ~= "number" then
1760
e72f9eac51c8 mod_storage_(various): Order swapping in 54c8a0cb2996 was backwards
Kim Alvefur <zash@zash.se>
parents: 1759
diff changeset
25 when, with, data = data, when, with;
1753
54c8a0cb2996 mod_storage_(archive-capable): Change order of arguments to :append to be the same as return values from :find iterator (see prosody 41725f3df3cc)
Kim Alvefur <zash@zash.se>
parents: 1752
diff changeset
26 end
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
27 if getmetatable(data) ~= st.stanza_mt then
1739
940a4ab75cec mod_storage_xmlarchive: This module only supports storage of stanzas, log a warning about attempts to do otherwise
Kim Alvefur <zash@zash.se>
parents: 1738
diff changeset
28 module:log("error", "Attempt to store non-stanza object, traceback: %s", debug.traceback());
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
29 return nil, "unsupported-datatype";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
30 end
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
31
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
32 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
33 data = tostring(data) .. "\n";
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
34
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
35 local day = dt.date(when);
1969
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
36 local ok, err = dm.append_raw(username.."@"..day, module.host, self.store, "xml", data);
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
37 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
38 return nil, err;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
39 end
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
40
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
41 -- If the day-file is missing then we need to add it to the list of days
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
42 local first_of_day = not lfs.attributes(dm.getpath(username .. "@" .. day, module.host, self.store, "list"));
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
43
1969
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
44 local offset = ok and err;
1740
11f7fb2f927f mod_storage_xmlarchive: Code is annoying to read when every other line is 'if not ok then return'
Kim Alvefur <zash@zash.se>
parents: 1739
diff changeset
45
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
46 local id = day .. "-" .. hmac_sha256(username.."@"..day.."+"..offset, data, true):sub(-16);
1742
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
47 ok, err = dm.list_append(username.."@"..day, module.host, self.store, { id = id, when = dt.datetime(when), with = with, offset = offset, length = #data });
2290
4786bf0a9334 mod_storage_xmlarchive: Determine if a message is the first of day by checking if the list file exists before
Kim Alvefur <zash@zash.se>
parents: 2289
diff changeset
48 if ok and first_of_day then
2038
a85b5c3791dc mod_storage_xmlarchive: Collect return value when adding to date index
Kim Alvefur <zash@zash.se>
parents: 1969
diff changeset
49 ok, err = dm.list_append(username, module.host, self.store, day);
1969
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
50 end
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
51 if not ok then
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
52 return nil, err;
e63dba236a2a mod_storage_xmlarchive: Use datamanager.append_raw (had that code duplicated here)
Kim Alvefur <zash@zash.se>
parents: 1831
diff changeset
53 end
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
54 return id;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
55 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
56
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
57 function archive:find(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
58 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
59 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
60
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
61 local result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
62 local function cb(_, stanza)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
63 if result then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
64 module:log("warn", "Multiple items in chunk");
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
65 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
66 result = stanza;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
67 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
68
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
69 local stream_sess = { notopen = true };
2183
13a8bbf256dd mod_storage_xmlarchive: Pass a default namespace to xmppstream so result items default to xmlns = nil (fixes non-mam use, like mod_mam_muc sending history)
Kim Alvefur <zash@zash.se>
parents: 2054
diff changeset
70 local stream = new_stream(stream_sess, { handlestanza = cb, stream_ns = "jabber:client", default_ns = "jabber:client" });
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
71 local dates = dm.list_load(username, module.host, self.store) or empty;
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
72 local function reset_stream()
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
73 stream:reset();
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
74 stream_sess.notopen = true;
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
75 stream:feed(st.stanza("stream", { xmlns = "jabber:client" }):top_tag());
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
76 stream_sess.notopen = nil;
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
77 end
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
78 reset_stream();
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
79
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
80 local limit = query.limit;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
81 local start_day, step, last_day = 1, 1, #dates;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
82 local count = 0;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
83 local rev = query.reverse;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
84 local in_range = not (query.after or query.before);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
85 if query.after or query.start then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
86 local d = query.after and query.after:sub(1, 10) or dt.date(query.start);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
87 for i = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
88 if dates[i] == d then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
89 start_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
90 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
91 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
92 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
93 if query.before or query["end"] then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
94 local d = query.before and query.before:sub(1, 10) or dt.date(query["end"]);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
95 for i = #dates, 1, -1 do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
96 if dates[i] == d then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
97 last_day = i; break;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
98 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
99 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
100 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
101 if rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
102 start_day, step, last_day = last_day, -step, start_day;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
103 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
104 local items, xmlfile;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
105 local first_item, last_item;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
106
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
107 return function ()
2054
b7c528027762 mod_storage_xmlarchive: Only try to close xmlfile if it has been opened (fixes traceback with limit=0)
Kim Alvefur <zash@zash.se>
parents: 2041
diff changeset
108 if limit and count >= limit then if xmlfile then xmlfile:close() end return; end
1726
160c35d2a5a2 mod_storage_xmlarchive: Improve logging of parse errors
Kim Alvefur <zash@zash.se>
parents: 1690
diff changeset
109 local filename;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
110
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
111 for d = start_day, last_day, step do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
112 if d ~= start_day or not items then
1728
a51beeb5aecf mod_storage_xmlarchive: Adjust wording in log message
Kim Alvefur <zash@zash.se>
parents: 1727
diff changeset
113 module:log("debug", "Loading items from %s", dates[d]);
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
114 start_day = d;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
115 items = dm.list_load(username .. "@" .. dates[d], module.host, self.store) or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
116 if not rev then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
117 first_item, last_item = 1, #items;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
118 else
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
119 first_item, last_item = #items, 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
120 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
121 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
122
1741
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
123 local q_with, q_start, q_end = query.with, query.start, query["end"];
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
124 for i = first_item, last_item, step do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
125 local item = items[i];
2396
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
126 if not item then
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
127 module:log("warn", "data[%q][%d] is nil", dates[d], i);
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
128 break;
544ee057a23f mod_storage_xmlarchive: Check if item exists before checking if its fields
Kim Alvefur <zash@zash.se>
parents: 2290
diff changeset
129 end
1741
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
130 local i_when, i_with = item.when, item.with;
1742
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
131 if type(i_when) == "string" then
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
132 i_when = dt.parse(i_when);
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
133 end
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
134 if type(i_when) ~= "number" then
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
135 module:log("warn", "data[%q][%d].when is invalid", dates[d], i);
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
136 break;
ec50cecc9318 mod_storage_xmlarchive: Store timestamps in text form, I don't trust numeric timestamps in Lua
Kim Alvefur <zash@zash.se>
parents: 1741
diff changeset
137 end
2397
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
138 if not xmlfile then
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
139 local ferr;
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
140 filename = dm.getpath(username .. "@" .. dates[d], module.host, self.store, "xml");
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
141 xmlfile, ferr = io.open(filename);
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
142 if not xmlfile then
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
143 module:log("error", "Error: %s", ferr);
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
144 return;
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
145 end
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
146 end
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
147 if in_range
1741
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
148 and (not q_with or i_with == q_with)
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
149 and (not q_start or i_when >= q_start)
07ceaf5f6f0d mod_storage_xmlarchive: Optimize access to variables used in inner loop
Kim Alvefur <zash@zash.se>
parents: 1740
diff changeset
150 and (not q_end or i_when <= q_end) then
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
151 count = count + 1;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
152 first_item = i + step;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
153
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
154 xmlfile:seek("set", item.offset);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
155 local data = xmlfile:read(item.length);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
156 local ok, err = stream:feed(data);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
157 if not ok then
1726
160c35d2a5a2 mod_storage_xmlarchive: Improve logging of parse errors
Kim Alvefur <zash@zash.se>
parents: 1690
diff changeset
158 module:log("warn", "Parse error in %s at %d+%d: %s", filename, item.offset, item.length, err);
1727
8f12afb633ec mod_storage_xmlarchive: Attempt to recover after parse failures
Kim Alvefur <zash@zash.se>
parents: 1726
diff changeset
159 reset_stream();
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
160 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
161 if result then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
162 local stanza = result;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
163 result = nil;
1752
3f3689a16133 mod_storage_xmlarchive: Return 'when' as number
Kim Alvefur <zash@zash.se>
parents: 1742
diff changeset
164 return item.id, stanza, i_when, i_with;
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
165 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
166 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
167 if (rev and item.id == query.after) or
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
168 (not rev and item.id == query.before) then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
169 in_range = false;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
170 limit = count;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
171 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
172 if (rev and item.id == query.before) or
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
173 (not rev and item.id == query.after) then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
174 in_range = true;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
175 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
176 end
2397
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
177 if xmlfile then
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
178 xmlfile:close();
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
179 xmlfile = nil;
7f9bf161f640 mod_storage_xmlarchive: Open XML file later, just before it is needed
Kim Alvefur <zash@zash.se>
parents: 2396
diff changeset
180 end
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
181 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
182 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
183 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
184
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
185 function archive:delete(username, query)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
186 username = username or "@";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
187 query = query or empty;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
188 if query.with or query.start or query.after then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
189 return nil, "not-implemented"; -- Only trimming the oldest messages
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
190 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
191 local before = query.before or query["end"] or "9999-12-31";
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
192 if type(before) == "number" then before = dt.date(before); else before = before:sub(1, 10); end
2040
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
193 local dates, err = dm.list_load(username, module.host, self.store);
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
194 if not dates or next(dates) == nil then
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
195 if not err then return true end -- already empty
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
196 return dates, err;
459e1878d23c mod_storage_xmlarchive: Return earlier if attempting to delete from empty archive
Kim Alvefur <zash@zash.se>
parents: 2038
diff changeset
197 end
2041
7c61ab512d0b mod_storage_xmlarchive: Return earlier if attempting to delete messages older than the oldest existing
Kim Alvefur <zash@zash.se>
parents: 2040
diff changeset
198 if dates[1] > before then return true; end -- Nothing to delete
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
199 local remaining_dates = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
200 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
201 if dates[d] >= before then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
202 table.insert(remaining_dates, dates[d]);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
203 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
204 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
205 table.sort(remaining_dates);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
206 local ok, err = dm.list_store(username, module.host, self.store, remaining_dates);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
207 if not ok then return ok, err; end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
208 for d = 1, #dates do
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
209 if dates[d] < before then
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
210 os.remove(dm.getpath(username .. "@" .. dates[d], module.host, self.store, "list"));
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
211 os.remove(dm.getpath(username .. "@" .. dates[d], module.host, self.store, "xml"));
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
212 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
213 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
214 return true;
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
215 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
216
1831
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
217 function archive:dates(username)
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
218 return dm.list_load(username, module.host, self.store);
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
219 end
004d3bfc05ea mod_storage_xmlarchive: Expose method for getting a list of dates which do have messages
Kim Alvefur <zash@zash.se>
parents: 1819
diff changeset
220
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
221 local provider = {};
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
222 function provider:open(store, typ)
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
223 if typ ~= "archive" then return nil, "unsupported-store"; end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
224 return setmetatable({ store = store }, archive_mt);
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
225 end
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
226
1819
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
227 function provider:purge(username)
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
228 for store in dm.stores(username, module.host) do
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
229 local dates = dm.list_load(username, module.host, store) or empty;
2271
b34f4591366a mod_storage_xmlarchive: Fix date pattern in purge (related to #725)
Kim Alvefur <zash@zash.se>
parents: 2183
diff changeset
230 if dates[1] and type(dates[1]) == "string" and dates[1]:match("^%d%d%d%d%-%d%d%-%d%d$") then
1819
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
231 module:log("info", "Store %s looks like an archive store, emptying it...", store);
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
232 provider:open(store, "archive"):delete(username);
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
233 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
234 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
235 return true;
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
236 end
1b08597b5e6f mod_storage_xmlarchive: Add support for purging (used when deleting users)
Kim Alvefur <zash@zash.se>
parents: 1793
diff changeset
237
1690
8c0fbc685364 mod_storage_xmlarchive: New stanza archive-only storage module backed by plain text files
Kim Alvefur <zash@zash.se>
parents:
diff changeset
238 module:provides("storage", provider);