Annotate

mod_storage_lmdb/mod_storage_lmdb.lua @ 1981:1f815f57fa57

mod_storage_lmdb: Factor out a transaction wrapper
author Kim Alvefur <zash@zash.se>
date Fri, 18 Dec 2015 22:00:00 +0100
parent 1800:669d1208221a
child 1982:bb0b2eae5563
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
1 -- mod_storage_lmdb
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
2 -- Copyright (C) 2015 Kim Alvefur
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
3 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
4 -- This file is MIT/X11 licensed.
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
5 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
6 -- Depends on lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
7 -- https://github.com/shmul/lightningdbm
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
8 --
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 -- luacheck: globals prosody open
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
11 local assert = assert;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
12 local select = select;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
13 local xpcall = xpcall;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
14 local traceback = debug.traceback;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
15
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
16 local lmdb = require"lightningmdb";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
17 local lfs = require"lfs";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
18 local path = require"util.paths";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
19 local serialization = require"util.serialization";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
20 local serialize = serialization.serialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
21 local deserialize = serialization.deserialize;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
23 local function transaction(env, func, ...)
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
24 local args, n_args = {...}, select("#", ...);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
25 local t = env:txn_begin(nil, 0);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
26 local function f() return func(t, unpack(args, 1, n_args)); end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
27 local success, a, b, c = xpcall(f, traceback);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
28 if not success then
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
29 io.stderr:write(a, "\n\n");
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
30 t:abort();
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
31 os.exit()
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
32 return success, a;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
33 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
34 local ok, err = t:commit();
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
35 if not ok then
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
36 return ok, err;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
37 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
38 return success, a, b, c;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
39 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
40
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
41 local function keyvalue_set(t, db, key, value)
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
42 if value ~= nil then
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
43 return assert(t:put(db, key, value, 0));
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
44 else
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
45 return t:del(db, key, value);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
46 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
47 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
48
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
49 local function keyvalue_get(t, db, key)
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
50 local data, err = t:get(db, key, 0);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
51 assert(data or not err, err);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
52 return data;
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
53 end
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
54
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
55 local drivers = {};
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
56 local provider = {};
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
57
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
58 local keyval = {};
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
59 local keyval_mt = { __index = keyval, flags = lmdb.MDB_CREATE };
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
60 drivers.keyval = keyval_mt;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
61
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
62 function keyval:set(user, value)
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
63 if type(value) == "table" and next(value) == nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
64 value = nil;
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
65 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
66 if value ~= nil then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
67 value = serialize(value);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
68 end
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
69 return transaction(self.env, keyvalue_set, self.db, user, value);
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
70 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
71
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
72 function keyval:get(user)
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
73 local ok, data = transaction(self.env, keyvalue_get, self.db, user);
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
74 if not ok then return ok, data; end
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
75 return deserialize(data);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
76 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
77
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
78 function provider:init(config)
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
79 if config.base_path then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
80 lfs.mkdir(config.base_path);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
81 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
82 local env = lmdb.env_create();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
83 env:set_maxdbs(config.maxdbs or 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
84 local env_flags = 0;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
85 if config.flags then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
86 for flag in config.flags do
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
87 env_flags = env_flags + assert(lmdb["MDB_"..flag:upper()], "No such flag "..flag);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
88 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
89 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
90 env:open(config.base_path or ".", env_flags, tonumber("640", 8));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
91 self.env = env;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
92 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
93
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
94 function provider:open(store, typ)
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
95 typ = typ or "keyval";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
96 local driver_mt = drivers[typ];
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
97 if not driver_mt then
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
98 return nil, "unsupported-store";
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
99 end
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
100 local env = self.env;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
101 local t = env:txn_begin(nil, 0);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
102 local db = t:dbi_open(store.."_"..typ, driver_mt.flags);
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
103 local ok, err = t:commit();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
104 if not ok then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
105 module:log("error", "Could not open database %s_%s: %s", store, typ, tostring(err));
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
106 return ok, err;
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
107 end
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
108
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
109 return setmetatable({ env = env, store = store, type = typ, db = db }, driver_mt);
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
110 end
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
111
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
112 if prosody then
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
113 provider:init({
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
114 base_path = path.resolve_relative_path(prosody.paths.data, module.host);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
115 flags = module:get_option_set("lmdb_flags", {});
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
116 maxdbs = module:get_option_number("lmdb_maxdbs", 20);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
117 });
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
118
1981
1f815f57fa57 mod_storage_lmdb: Factor out a transaction wrapper
Kim Alvefur <zash@zash.se>
parents: 1800
diff changeset
119 function module.unload() --luacheck: ignore
1799
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
120 provider.env:sync(1);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
121 provider.env:close();
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
122 end
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
123
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
124 module:provides("storage", provider);
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
125 else
d2dd1db9ece6 mod_storage_lmdb: Rearrange module to allow using as a library
Kim Alvefur <zash@zash.se>
parents: 1755
diff changeset
126 return provider;
1755
0a21b16b9075 mod_storage_lmdb: Storage module using Lightning Memory-Mapped Database
Kim Alvefur <zash@zash.se>
parents:
diff changeset
127 end