Software / code / prosody-modules
Annotate
mod_storage_multi/mod_storage_multi.lua @ 5571:ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
This should ensure that an entry that has a publish timestmap after the
previously oldest post, but before the time of the last poll check, is
published to the node.
Previously if an entry would be skipped if it was published at 13:00
with a timestamp of 12:30, where the last poll was at 12:45.
For feeds that lack a timestamp, it now looks for the first post that is
not published, assuming that the feed is in reverse chronological order,
then iterates back up from there.
| author | Kim Alvefur <zash@zash.se> |
|---|---|
| date | Sun, 25 Jun 2023 16:27:55 +0200 |
| parent | 1492:8c4a2d85e8bf |
| rev | line source |
|---|---|
|
1492
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 -- mod_storage_multi |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
3 local storagemanager = require"core.storagemanager"; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
4 local backends = module:get_option_array(module.name); -- TODO better name? |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
6 -- TODO migrate data "upwards" |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
7 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
8 -- one → one successful write is success |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
9 -- all → all backends must report success |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
10 -- majority → majority of backends must report success |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
11 local policy = module:get_option_string(module.name.."_policy", "all"); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
12 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
13 local keyval_store = {}; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
14 keyval_store.__index = keyval_store; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
15 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
16 function keyval_store:get(username) |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
17 local backends = self.backends; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
18 local data, err; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
19 for i = 1, #backends do |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
20 module:log("debug", "%s:%s:get(%q)", tostring(backends[i].get), backends[i]._store, username); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
21 data, err = backends[i]:get(username); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
22 if err then |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
23 module:log("error", tostring(err)); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
24 elseif not data then |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
25 module:log("debug", "No data returned"); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
26 else |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
27 module:log("debug", "Data returned"); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
28 return data, err; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
29 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
30 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
31 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
32 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
33 -- This is where it gets complicated |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
34 function keyval_store:set(username, data) |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
35 local backends = self.backends; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
36 local ok, err, backend; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
37 local all, one, oks = true, false, 0; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
38 for i = 1, #backends do |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
39 backend = backends[i]; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
40 module:log("debug", "%s:%s:set(%q)", tostring(backends[i].get), backends[i].store, username); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
41 ok, err = backend:set(username, data); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
42 if not ok then |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
43 module:log("error", "Error in storage driver %s: %s", backend.name, tostring(err)); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
44 else |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
45 oks = oks + 1; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
46 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
47 one = one or ok; -- At least one successful write |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
48 all = all and ok; -- All successful |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
49 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
50 if policy == "all" then |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
51 return all, err |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
52 elseif policy == "majority" then |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
53 return oks > (#backends/2), err; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
54 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
55 -- elseif policy == "one" then |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
56 return one, err; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
57 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
58 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
59 local stores = { |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
60 keyval = keyval_store; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
61 } |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
62 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
63 local driver = {}; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
64 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
65 function driver:open(store, typ) |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
66 local store_mt = stores[typ or "keyval"]; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
67 if store_mt then |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
68 local my_backends = {}; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
69 local driver, opened |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
70 for i = 1, #backends do |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
71 driver = storagemanager.load_driver(module.host, backends[i]); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
72 opened = driver:open(store, typ); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
73 my_backends[i] = assert(driver:open(store, typ)); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
74 my_backends[i]._store = store; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
75 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
76 return setmetatable({ backends = my_backends }, store_mt); |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
77 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
78 return nil, "unsupported-store"; |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
79 end |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
80 |
|
8c4a2d85e8bf
mod_storage_multi: Multi-backend storage module (WIP)
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
81 module:provides("storage", driver); |
