prosody
dbd7a6b09ada
util/datamanager.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Annotate

util/datamanager.lua @ 13235:dbd7a6b09ada

util.datamanager: Close file handle when done using it It gets closed eventually but at high load they could potentially lead to reaching FD limits faster.
author Kim Alvefur <zash@zash.se>
date Fri, 21 Jul 2023 18:28:54 +0200
parent 13234:c886ea087b38
child 13236:9c72f93b7a02
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1523
841d61be198f Remove version number from copyright headers
Matthew Wild <mwild1@gmail.com>
parents: 1462
diff changeset
1 -- Prosody IM
2923
b7049746bd29 Update copyright headers for 2010
Matthew Wild <mwild1@gmail.com>
parents: 2159
diff changeset
2 -- Copyright (C) 2008-2010 Matthew Wild
b7049746bd29 Update copyright headers for 2010
Matthew Wild <mwild1@gmail.com>
parents: 2159
diff changeset
3 -- Copyright (C) 2008-2010 Waqas Hussain
4452
7de17ca4de14 util/datamanager: Use package.config to figure out directory seperator
James Callahan <james@chatid.com>
parents: 4112
diff changeset
4 --
758
b1885732e979 GPL->MIT!
Matthew Wild <mwild1@gmail.com>
parents: 732
diff changeset
5 -- This project is MIT/X11 licensed. Please see the
b1885732e979 GPL->MIT!
Matthew Wild <mwild1@gmail.com>
parents: 732
diff changeset
6 -- COPYING file in the source package for more information.
519
cccd610a0ef9 Insert copyright/license headers
Matthew Wild <mwild1@gmail.com>
parents: 456
diff changeset
7 --
cccd610a0ef9 Insert copyright/license headers
Matthew Wild <mwild1@gmail.com>
parents: 456
diff changeset
8
cccd610a0ef9 Insert copyright/license headers
Matthew Wild <mwild1@gmail.com>
parents: 456
diff changeset
9
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
10 local string = string;
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
11 local format = string.format;
5051
71253db26fda util.datamanager: Remove a few unused imports
Kim Alvefur <zash@zash.se>
parents: 5049
diff changeset
12 local setmetatable = setmetatable;
71253db26fda util.datamanager: Remove a few unused imports
Kim Alvefur <zash@zash.se>
parents: 5049
diff changeset
13 local ipairs = ipairs;
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
14 local char = string.char;
5021
85b2689dbcfe Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents: 4452
diff changeset
15 local pcall = pcall;
12975
d10957394a3c util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 12388
diff changeset
16 local log = require "prosody.util.logger".init("datamanager");
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
17 local io_open = io.open;
206
e30d0e30a0ff Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents: 182
diff changeset
18 local os_remove = os.remove;
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
19 local os_rename = os.rename;
5051
71253db26fda util.datamanager: Remove a few unused imports
Kim Alvefur <zash@zash.se>
parents: 5049
diff changeset
20 local tonumber = tonumber;
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
21 local floor = math.floor;
206
e30d0e30a0ff Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents: 182
diff changeset
22 local next = next;
7432
92f721226753 util.datamanager: Import tostring and type (fix global access)
Kim Alvefur <zash@zash.se>
parents: 7202
diff changeset
23 local type = type;
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
24 local t_insert = table.insert;
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
25 local t_concat = table.concat;
12975
d10957394a3c util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 12388
diff changeset
26 local envloadfile = require"prosody.util.envload".envloadfile;
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
27 local envload = require"prosody.util.envload".envload;
12975
d10957394a3c util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 12388
diff changeset
28 local serialize = require "prosody.util.serialization".serialize;
3086
931acb1188b1 util.datamanager: When failing to load a file, and the file exists, return nil, error.
Waqas Hussain <waqas20@gmail.com>
parents: 2925
diff changeset
29 local lfs = require "lfs";
9993
02a41315d275 Fix various spelling mistakes [codespell]
Kim Alvefur <zash@zash.se>
parents: 8555
diff changeset
30 -- Extract directory separator from package.config (an undocumented string that comes with lua)
7995
b99404a340fc util.datamanager: Split long line [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7929
diff changeset
31 local path_separator = assert ( package.config:match ( "^([^\n]+)" ) , "package.config not in standard form" )
b99404a340fc util.datamanager: Split long line [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7929
diff changeset
32
4093
36555949bd16 util.datamanager: Use prosody.paths.data as the initial value for data_path
Matthew Wild <mwild1@gmail.com>
parents: 3723
diff changeset
33 local prosody = prosody;
2444
267d6482bac6 util.datamanager: Use pposix.mkdir if available
Matthew Wild <mwild1@gmail.com>
parents: 2328
diff changeset
34
13234
c886ea087b38 util.datamanager: Disable blockwise removal
Kim Alvefur <zash@zash.se>
parents: 13231
diff changeset
35 --luacheck: ignore 211/blocksize 211/remove_blocks
13186
affaf6d08d26 util.datamanager: Pad list writes to avoid crossing block boundaries
Kim Alvefur <zash@zash.se>
parents: 13185
diff changeset
36 local blocksize = 0x1000;
5118
0dc9e6c128c3 util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents: 5103
diff changeset
37 local raw_mkdir = lfs.mkdir;
8011
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
38 local atomic_append;
13185
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
39 local remove_blocks;
7925
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
40 local ENOENT = 2;
5118
0dc9e6c128c3 util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents: 5103
diff changeset
41 pcall(function()
12975
d10957394a3c util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 12388
diff changeset
42 local pposix = require "prosody.util.pposix";
5118
0dc9e6c128c3 util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents: 5103
diff changeset
43 raw_mkdir = pposix.mkdir or raw_mkdir; -- Doesn't trample on umask
8011
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
44 atomic_append = pposix.atomic_append;
13234
c886ea087b38 util.datamanager: Disable blockwise removal
Kim Alvefur <zash@zash.se>
parents: 13231
diff changeset
45 -- remove_blocks = pposix.remove_blocks;
7927
7132abcf669e util.datamanager: Import value of ENOENT from util.pposix (if available)
Kim Alvefur <zash@zash.se>
parents: 7925
diff changeset
46 ENOENT = pposix.ENOENT or ENOENT;
5118
0dc9e6c128c3 util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents: 5103
diff changeset
47 end);
5045
4ba6940deed0 util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents: 5038
diff changeset
48
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
49 local _ENV = nil;
8555
4f0f5b49bb03 vairious: Add annotation when an empty environment is set [luacheck]
Kim Alvefur <zash@zash.se>
parents: 8299
diff changeset
50 -- luacheck: std none
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
51
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
52 ---- utils -----
8299
756a2a00e7e7 util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
53 local encode, decode, store_encode;
2444
267d6482bac6 util.datamanager: Use pposix.mkdir if available
Matthew Wild <mwild1@gmail.com>
parents: 2328
diff changeset
54 do
7732
aba3dd84d9f0 util.datamanager: Use the 'base' argument to tonumber() to indicate hexadecimal
Kim Alvefur <zash@zash.se>
parents: 7674
diff changeset
55 local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber(k, 16)); return t[k]; end });
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
56
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
57 decode = function (s)
7734
6a52415ed68a util.datamanager: Shorter pattern
Kim Alvefur <zash@zash.se>
parents: 7733
diff changeset
58 return s and (s:gsub("%%(%x%x)", urlcodes));
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
59 end
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
60
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
61 encode = function (s)
628
3712d36b6d25 Fixed URL encoding to generate %0x instead of %x
Waqas Hussain <waqas20@gmail.com>
parents: 615
diff changeset
62 return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end));
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
63 end
8299
756a2a00e7e7 util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
64
756a2a00e7e7 util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
65 -- Special encode function for store names, which historically were unencoded.
756a2a00e7e7 util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
66 -- All currently known stores use a-z and underscore, so this one preserves underscores.
756a2a00e7e7 util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
67 store_encode = function (s)
756a2a00e7e7 util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
68 return s and (s:gsub("[^_%w]", function (c) return format("%%%02x", c:byte()); end));
756a2a00e7e7 util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
69 end
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
70 end
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
71
8011
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
72 if not atomic_append then
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
73 function atomic_append(f, data)
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
74 local pos = f:seek();
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
75 if not f:write(data) or not f:flush() then
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
76 f:seek("set", pos);
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
77 f:write((" "):rep(#data));
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
78 f:flush();
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
79 return nil, "write-failed";
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
80 end
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
81 return true;
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
82 end
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
83 end
f8ba814fe029 util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents: 7996
diff changeset
84
643
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
85 local _mkdir = {};
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
86 local function mkdir(path)
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
87 path = path:gsub("/", path_separator); -- TODO as an optimization, do this during path creation rather than here
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
88 if not _mkdir[path] then
2444
267d6482bac6 util.datamanager: Use pposix.mkdir if available
Matthew Wild <mwild1@gmail.com>
parents: 2328
diff changeset
89 raw_mkdir(path);
643
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
90 _mkdir[path] = true;
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
91 end
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
92 return path;
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
93 end
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
94
4108
e3e3aa286334 util.datamanager: Handle gracefully the lack of prosody.paths.data
Matthew Wild <mwild1@gmail.com>
parents: 4093
diff changeset
95 local data_path = (prosody and prosody.paths and prosody.paths.data) or ".";
1381
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
96 local callbacks = {};
643
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
97
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
98 ------- API -------------
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
99
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
100 local function set_data_path(path)
1097
c5b33640a5f0 util.datamanager: Lower log level of 'Setting data path' to 'debug'
Matthew Wild <mwild1@gmail.com>
parents: 932
diff changeset
101 log("debug", "Setting data path to: %s", path);
452
613c5c6bdce4 Added option core.data_path
Waqas Hussain <waqas20@gmail.com>
parents: 267
diff changeset
102 data_path = path;
613c5c6bdce4 Added option core.data_path
Waqas Hussain <waqas20@gmail.com>
parents: 267
diff changeset
103 end
1381
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
104
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
105 local function callback(username, host, datastore, data)
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
106 for _, f in ipairs(callbacks) do
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
107 username, host, datastore, data = f(username, host, datastore, data);
1462
44780b856ce7 datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents: 1381
diff changeset
108 if username == false then break; end
1381
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
109 end
4452
7de17ca4de14 util/datamanager: Use package.config to figure out directory seperator
James Callahan <james@chatid.com>
parents: 4112
diff changeset
110
1381
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
111 return username, host, datastore, data;
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
112 end
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
113 local function add_callback(func)
1381
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
114 if not callbacks[func] then -- Would you really want to set the same callback more than once?
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
115 callbacks[func] = true;
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
116 callbacks[#callbacks+1] = func;
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
117 return true;
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
118 end
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
119 end
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
120 local function remove_callback(func)
1381
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
121 if callbacks[func] then
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
122 for i, f in ipairs(callbacks) do
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
123 if f == func then
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
124 callbacks[i] = nil;
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
125 callbacks[f] = nil;
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
126 return true;
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
127 end
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
128 end
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
129 end
932
1ed3e5fe165a Added: datamanager: Allow a callback to be installed which selectively prevents disk writes
Waqas Hussain <waqas20@gmail.com>
parents: 915
diff changeset
130 end
452
613c5c6bdce4 Added option core.data_path
Waqas Hussain <waqas20@gmail.com>
parents: 267
diff changeset
131
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
132 local function getpath(username, host, datastore, ext, create)
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
133 ext = ext or "dat";
2328
2804d81206d8 util.datamanager: Store data stores with no host in '_global' folder
Matthew Wild <mwild1@gmail.com>
parents: 2159
diff changeset
134 host = (host and encode(host)) or "_global";
643
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
135 username = username and encode(username);
8299
756a2a00e7e7 util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents: 8092
diff changeset
136 datastore = store_encode(datastore);
84
d0a0bac6815e Added: Datastore support for hosts and global data in addition to users
Waqas Hussain <waqas20@gmail.com>
parents: 0
diff changeset
137 if username then
643
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
138 if create then mkdir(mkdir(mkdir(data_path).."/"..host).."/"..datastore); end
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
139 return format("%s/%s/%s/%s.%s", data_path, host, datastore, username, ext);
5243
07e8256efcda util.datamanager: Remove dead code path
Kim Alvefur <zash@zash.se>
parents: 5153
diff changeset
140 else
643
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
141 if create then mkdir(mkdir(data_path).."/"..host); end
8ff454831f7d Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents: 628
diff changeset
142 return format("%s/%s/%s.%s", data_path, host, datastore, ext);
84
d0a0bac6815e Added: Datastore support for hosts and global data in addition to users
Waqas Hussain <waqas20@gmail.com>
parents: 0
diff changeset
143 end
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
144 end
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
145
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
146 local function load(username, host, datastore)
7925
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
147 local data, err, errno = envloadfile(getpath(username, host, datastore), {});
117
8e5c5e6a3240 Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents: 88
diff changeset
148 if not data then
7925
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
149 if errno == ENOENT then
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
150 -- No such file, ok to ignore
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
151 return nil;
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
152 end
7996
f4c0fb54e16c util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents: 7995
diff changeset
153 log("error", "Failed to load %s storage ('%s') for user: %s@%s", datastore, err, username or "nil", host or "nil");
f4c0fb54e16c util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents: 7995
diff changeset
154 return nil, "Error reading storage";
117
8e5c5e6a3240 Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents: 88
diff changeset
155 end
5021
85b2689dbcfe Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents: 4452
diff changeset
156
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
157 local success, ret = pcall(data);
117
8e5c5e6a3240 Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents: 88
diff changeset
158 if not success then
4993
5243b74a4cbb Hopefully inert commit to clean up logging across a number of modules, removing all cases of concatenation when building log messages
Matthew Wild <mwild1@gmail.com>
parents: 4452
diff changeset
159 log("error", "Unable to load %s storage ('%s') for user: %s@%s", datastore, ret, username or "nil", host or "nil");
3086
931acb1188b1 util.datamanager: When failing to load a file, and the file exists, return nil, error.
Waqas Hussain <waqas20@gmail.com>
parents: 2925
diff changeset
160 return nil, "Error reading storage";
117
8e5c5e6a3240 Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents: 88
diff changeset
161 end
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
162 return ret;
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
163 end
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
164
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
165 local function atomic_store(filename, data)
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
166 local scratch = filename.."~";
10531
810abf34919a util.datamanager: Ignore unused 'errno' variable [luacheck]
Kim Alvefur <zash@zash.se>
parents: 9993
diff changeset
167 local f, ok, msg, errno; -- luacheck: ignore errno
810abf34919a util.datamanager: Ignore unused 'errno' variable [luacheck]
Kim Alvefur <zash@zash.se>
parents: 9993
diff changeset
168 -- TODO return util.error with code=errno?
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
169
7925
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
170 f, msg, errno = io_open(scratch, "w");
7202
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
171 if not f then
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
172 return nil, msg;
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
173 end
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
174
7202
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
175 ok, msg = f:write(data);
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
176 if not ok then
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
177 f:close();
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
178 os_remove(scratch);
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
179 return nil, msg;
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
180 end
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
181
7202
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
182 ok, msg = f:close();
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
183 if not ok then
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
184 os_remove(scratch);
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
185 return nil, msg;
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
186 end
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
187
7202
5bf0ff3882aa util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents: 7201
diff changeset
188 return os_rename(scratch, filename);
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
189 end
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
190
6575
bdaadf70a48f util.datamanager: Check that the global 'prosody' exists before using it (fixes nil indexing in use outside of prosody)
Kim Alvefur <zash@zash.se>
parents: 5441
diff changeset
191 if prosody and prosody.platform ~= "posix" then
5065
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
192 -- os.rename does not overwrite existing files on Windows
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
193 -- TODO We could use Transactional NTFS on Vista and above
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
194 function atomic_store(filename, data)
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
195 local f, err = io_open(filename, "w");
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
196 if not f then return f, err; end
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
197 local ok, msg = f:write(data);
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
198 if not ok then f:close(); return ok, msg; end
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
199 return f:close();
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
200 end
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
201 end
acfaf771f10e util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents: 5057
diff changeset
202
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
203 local function store(username, host, datastore, data)
206
e30d0e30a0ff Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents: 182
diff changeset
204 if not data then
e30d0e30a0ff Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents: 182
diff changeset
205 data = {};
e30d0e30a0ff Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents: 182
diff changeset
206 end
1381
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
207
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
208 username, host, datastore, data = callback(username, host, datastore, data);
1462
44780b856ce7 datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents: 1381
diff changeset
209 if username == false then
1381
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
210 return true; -- Don't save this data at all
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
211 end
46a58df8557d util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents: 1097
diff changeset
212
206
e30d0e30a0ff Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents: 182
diff changeset
213 -- save the datastore
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
214 local d = "return " .. serialize(data) .. ";\n";
5441
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
215 local mkdir_cache_cleared;
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
216 repeat
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
217 local ok, msg = atomic_store(getpath(username, host, datastore, nil, true), d);
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
218 if not ok then
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
219 if not mkdir_cache_cleared then -- We may need to recreate a removed directory
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
220 _mkdir = {};
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
221 mkdir_cache_cleared = true;
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
222 else
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
223 log("error", "Unable to write to %s storage ('%s') for user: %s@%s", datastore, msg, username or "nil", host or "nil");
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
224 return nil, "Error saving to storage";
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
225 end
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
226 end
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
227 if next(data) == nil then -- try to delete empty datastore
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
228 log("debug", "Removing empty %s datastore for user %s@%s", datastore, username or "nil", host or "nil");
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
229 os_remove(getpath(username, host, datastore));
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
230 end
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
231 -- we write data even when we are deleting because lua doesn't have a
12388
50fcd3879482 spelling: non-existing mistakes (thanks timeless)
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
232 -- platform independent way of checking for nonexisting files
5441
6a5c622cc6d4 util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents: 5440
diff changeset
233 until ok;
0
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
234 return true;
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
235 end
3e3171b59028 First commit, where do you want to go tomorrow?
matthew
parents:
diff changeset
236
6999
0ad66d12113a util.datamanager: Add some comments about the append function
Kim Alvefur <zash@zash.se>
parents: 6998
diff changeset
237 -- Append a blob of data to a file
6992
0622f2820d1d util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents: 6777
diff changeset
238 local function append(username, host, datastore, ext, data)
7000
2b57f77985a3 util.datamanager: Make sure only strings are passed as data to append()
Kim Alvefur <zash@zash.se>
parents: 6999
diff changeset
239 if type(data) ~= "string" then return; end
6994
507301531cf5 util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents: 6993
diff changeset
240 local filename = getpath(username, host, datastore, ext, true);
6996
644b1bddc676 util.datamanager: No shadowing of variable [luacheck]
Kim Alvefur <zash@zash.se>
parents: 6995
diff changeset
241
8013
72cfbe377326 util.datamanager: Rearrange locals
Kim Alvefur <zash@zash.se>
parents: 8011
diff changeset
242 local f = io_open(filename, "r+");
5066
4be7093edde9 util.datamanager: Try to open in read+write mode, then retry with write mode if that fails (usually because it doesn't exist)
Kim Alvefur <zash@zash.se>
parents: 5057
diff changeset
243 if not f then
7928
9d014aca766f util.datamanager: Use atomic store function when writing the first entry of a list
Kim Alvefur <zash@zash.se>
parents: 7927
diff changeset
244 return atomic_store(filename, data);
6999
0ad66d12113a util.datamanager: Add some comments about the append function
Kim Alvefur <zash@zash.se>
parents: 6998
diff changeset
245 -- File did probably not exist, let's create it
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
246 end
6999
0ad66d12113a util.datamanager: Add some comments about the append function
Kim Alvefur <zash@zash.se>
parents: 6998
diff changeset
247
5045
4ba6940deed0 util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents: 5038
diff changeset
248 local pos = f:seek("end");
13231
6a11d92ae436 util.datamanager: Disable block alignment
Kim Alvefur <zash@zash.se>
parents: 13186
diff changeset
249 --[[ TODO needs tests
13186
affaf6d08d26 util.datamanager: Pad list writes to avoid crossing block boundaries
Kim Alvefur <zash@zash.se>
parents: 13185
diff changeset
250 if (blocksize-(pos%blocksize)) < (#data%blocksize) then
affaf6d08d26 util.datamanager: Pad list writes to avoid crossing block boundaries
Kim Alvefur <zash@zash.se>
parents: 13185
diff changeset
251 -- pad to blocksize with newlines so that the next item is both on a new
affaf6d08d26 util.datamanager: Pad list writes to avoid crossing block boundaries
Kim Alvefur <zash@zash.se>
parents: 13185
diff changeset
252 -- block and a new line
affaf6d08d26 util.datamanager: Pad list writes to avoid crossing block boundaries
Kim Alvefur <zash@zash.se>
parents: 13185
diff changeset
253 atomic_append(f, ("\n"):rep(blocksize-(pos%blocksize)));
affaf6d08d26 util.datamanager: Pad list writes to avoid crossing block boundaries
Kim Alvefur <zash@zash.se>
parents: 13185
diff changeset
254 pos = f:seek("end");
affaf6d08d26 util.datamanager: Pad list writes to avoid crossing block boundaries
Kim Alvefur <zash@zash.se>
parents: 13185
diff changeset
255 end
13231
6a11d92ae436 util.datamanager: Disable block alignment
Kim Alvefur <zash@zash.se>
parents: 13186
diff changeset
256 --]]
6997
0ab228bc21c6 util.datamanager: Handle potential issues from fallocate
Kim Alvefur <zash@zash.se>
parents: 6996
diff changeset
257
8013
72cfbe377326 util.datamanager: Rearrange locals
Kim Alvefur <zash@zash.se>
parents: 8011
diff changeset
258 local ok, msg = atomic_append(f, data);
6997
0ab228bc21c6 util.datamanager: Handle potential issues from fallocate
Kim Alvefur <zash@zash.se>
parents: 6996
diff changeset
259
6998
86607fe755b6 util.datamanager: Handle potential error from :write() call
Kim Alvefur <zash@zash.se>
parents: 6997
diff changeset
260 if not ok then
86607fe755b6 util.datamanager: Handle potential error from :write() call
Kim Alvefur <zash@zash.se>
parents: 6997
diff changeset
261 f:close();
7001
2743759ca1b5 util.datamanager: Return extra location info
Kim Alvefur <zash@zash.se>
parents: 7000
diff changeset
262 return ok, msg, "write";
6992
0622f2820d1d util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents: 6777
diff changeset
263 end
6994
507301531cf5 util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents: 6993
diff changeset
264
507301531cf5 util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents: 6993
diff changeset
265 ok, msg = f:close();
507301531cf5 util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents: 6993
diff changeset
266 if not ok then
8014
ff3787033abb util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents: 8013
diff changeset
267 return ok, msg, "close";
6994
507301531cf5 util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents: 6993
diff changeset
268 end
507301531cf5 util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents: 6993
diff changeset
269
7001
2743759ca1b5 util.datamanager: Return extra location info
Kim Alvefur <zash@zash.se>
parents: 7000
diff changeset
270 return true, pos;
6992
0622f2820d1d util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents: 6777
diff changeset
271 end
0622f2820d1d util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents: 6777
diff changeset
272
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
273 local index_fmt, index_item_size, index_magic;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
274 if string.packsize then
13137
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
275 index_fmt = "T"; -- offset to the end of the item, length can be derived from two index items
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
276 index_item_size = string.packsize(index_fmt);
13137
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
277 index_magic = string.pack(index_fmt, 7767639 + 1); -- Magic string: T9 for "prosody", version number
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
278 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
279
6992
0622f2820d1d util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents: 6777
diff changeset
280 local function list_append(username, host, datastore, data)
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
281 if not data then return; end
1462
44780b856ce7 datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents: 1381
diff changeset
282 if callback(username, host, datastore) == false then return true; end
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
283 -- save the datastore
6992
0622f2820d1d util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents: 6777
diff changeset
284
6993
dc0c6b8dc638 util.datamanager: Overwrite 'data' variable instead of shadownig it [luacheck]
Kim Alvefur <zash@zash.se>
parents: 6992
diff changeset
285 data = "item(" .. serialize(data) .. ");\n";
8014
ff3787033abb util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents: 8013
diff changeset
286 local ok, msg, where = append(username, host, datastore, "list", data);
6992
0622f2820d1d util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents: 6777
diff changeset
287 if not ok then
8014
ff3787033abb util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents: 8013
diff changeset
288 log("error", "Unable to write to %s storage ('%s' in %s) for user: %s@%s",
ff3787033abb util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents: 8013
diff changeset
289 datastore, msg, where, username or "nil", host or "nil");
5045
4ba6940deed0 util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents: 5038
diff changeset
290 return ok, msg;
4ba6940deed0 util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents: 5038
diff changeset
291 end
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
292 if string.packsize then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
293 local offset = type(msg) == "number" and msg or 0;
13137
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
294 local index_entry = string.pack(index_fmt, offset + #data);
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
295 if offset == 0 then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
296 index_entry = index_magic .. index_entry;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
297 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
298 local ok, off = append(username, host, datastore, "lidx", index_entry);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
299 off = off or 0;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
300 -- If this was the first item, then both the data and index offsets should
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
301 -- be zero, otherwise there's some kind of mismatch and we should drop the
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
302 -- index and recreate it from scratch
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
303 -- TODO Actually rebuild the index in this case?
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
304 if not ok or (off == 0 and offset ~= 0) or (off ~= 0 and offset == 0) then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
305 os_remove(getpath(username, host, datastore, "lidx"));
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
306 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
307 end
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
308 return true;
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
309 end
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
310
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
311 local function list_store(username, host, datastore, data)
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
312 if not data then
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
313 data = {};
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
314 end
1462
44780b856ce7 datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents: 1381
diff changeset
315 if callback(username, host, datastore) == false then return true; end
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
316 -- save the datastore
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
317 local d = {};
8092
0a1c0f1107d2 util.datamanager: Use already known index instead of measuring length each iteration
Kim Alvefur <zash@zash.se>
parents: 8014
diff changeset
318 for i, item in ipairs(data) do
0a1c0f1107d2 util.datamanager: Use already known index instead of measuring length each iteration
Kim Alvefur <zash@zash.se>
parents: 8014
diff changeset
319 d[i] = "item(" .. serialize(item) .. ");\n";
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
320 end
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
321 os_remove(getpath(username, host, datastore, "lidx"));
5049
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
322 local ok, msg = atomic_store(getpath(username, host, datastore, "list", true), t_concat(d));
5d685f123332 util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents: 5045
diff changeset
323 if not ok then
4993
5243b74a4cbb Hopefully inert commit to clean up logging across a number of modules, removing all cases of concatenation when building log messages
Matthew Wild <mwild1@gmail.com>
parents: 4452
diff changeset
324 log("error", "Unable to write to %s storage ('%s') for user: %s@%s", datastore, msg, username or "nil", host or "nil");
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
325 return;
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
326 end
915
0fe5bf7ab81d util.datamanager: Don't delete data when first entry in table is 'false'. My favourite bug so far.
Matthew Wild <mwild1@gmail.com>
parents: 896
diff changeset
327 if next(data) == nil then -- try to delete empty datastore
1732
f1282fad2f99 datamanager: Fixed logging errors on deletion of datastores not owned by a user@host
Waqas Hussain <waqas20@gmail.com>
parents: 1523
diff changeset
328 log("debug", "Removing empty %s datastore for user %s@%s", datastore, username or "nil", host or "nil");
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
329 os_remove(getpath(username, host, datastore, "list"));
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
330 end
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
331 -- we write data even when we are deleting because lua doesn't have a
12388
50fcd3879482 spelling: non-existing mistakes (thanks timeless)
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
332 -- platform independent way of checking for nonexisting files
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
333 return true;
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
334 end
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
335
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
336 local function build_list_index(username, host, datastore, items)
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
337 log("debug", "Building index for (%s@%s/%s)", username, host, datastore);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
338 local filename = getpath(username, host, datastore, "list");
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
339 local fh, err, errno = io_open(filename);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
340 if not fh then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
341 return fh, err, errno;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
342 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
343 local prev_pos = 0; -- position before reading
13182
c48ae06e24d6 util.datamanager: Fix indexing first item if not at the very start
Kim Alvefur <zash@zash.se>
parents: 13181
diff changeset
344 local last_item_start = nil;
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
345
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
346 if items and items[1] then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
347 local last_item = items[#items];
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
348 last_item_start = fh:seek("set", last_item.start + last_item.length);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
349 else
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
350 items = {};
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
351 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
352
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
353 for line in fh:lines() do
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
354 if line:sub(1, 4) == "item" then
13182
c48ae06e24d6 util.datamanager: Fix indexing first item if not at the very start
Kim Alvefur <zash@zash.se>
parents: 13181
diff changeset
355 if prev_pos ~= 0 and last_item_start then
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
356 t_insert(items, { start = last_item_start; length = prev_pos - last_item_start });
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
357 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
358 last_item_start = prev_pos
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
359 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
360 -- seek position is at the start of the next line within each loop iteration
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
361 -- so we need to collect the "current" position at the end of the previous
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
362 prev_pos = fh:seek()
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
363 end
13235
dbd7a6b09ada util.datamanager: Close file handle when done using it
Kim Alvefur <zash@zash.se>
parents: 13234
diff changeset
364 fh:close();
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
365 if prev_pos ~= 0 then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
366 t_insert(items, { start = last_item_start; length = prev_pos - last_item_start });
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
367 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
368 return items;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
369 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
370
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
371 local function store_list_index(username, host, datastore, index)
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
372 local data = { index_magic };
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
373 for i, v in ipairs(index) do
13137
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
374 data[i + 1] = string.pack(index_fmt, v.start + v.length);
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
375 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
376 local filename = getpath(username, host, datastore, "lidx");
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
377 return atomic_store(filename, t_concat(data));
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
378 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
379
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
380 local index_mt = {
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
381 __index = function(t, i)
13137
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
382 if type(i) ~= "number" or i % 1 ~= 0 or i < 0 then
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
383 return
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
384 end
13137
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
385 if i <= 0 then
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
386 return 0
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
387 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
388 local fh = t.file;
13137
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
389 local pos = (i - 1) * index_item_size;
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
390 if fh:seek("set", pos) ~= pos then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
391 return nil
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
392 end
13137
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
393 local data = fh:read(index_item_size * 2);
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
394 if not data or #data ~= index_item_size * 2 then
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
395 return nil
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
396 end
13137
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
397 local start, next_pos = string.unpack(index_fmt .. index_fmt, data);
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
398 if pos == 0 then
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
399 start = 0
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
400 end
b417a49cc31b util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents: 13134
diff changeset
401 local length = next_pos - start;
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
402 local v = { start = start; length = length };
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
403 t[i] = v;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
404 return v;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
405 end;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
406 __len = function(t)
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
407 -- Account for both the header and the fence post error
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
408 return floor(t.file:seek("end") / index_item_size) - 1;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
409 end;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
410 }
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
411
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
412 local function get_list_index(username, host, datastore)
13180
48622b89f570 util.datamanager: Fix missing separator in log line
Kim Alvefur <zash@zash.se>
parents: 13137
diff changeset
413 log("debug", "Loading index for (%s@%s/%s)", username, host, datastore);
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
414 local index_filename = getpath(username, host, datastore, "lidx");
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
415 local ih = io_open(index_filename);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
416 if ih then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
417 local magic = ih:read(#index_magic);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
418 if magic ~= index_magic then
13181
87487056bccb util.datamanager: Reduce log level of left over debug messages to debug (thanks Trung)
Kim Alvefur <zash@zash.se>
parents: 13180
diff changeset
419 log("debug", "Index %q has wrong version number (got %q, expected %q), rebuilding...", index_filename, magic, index_magic);
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
420 -- wrong version or something
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
421 ih:close();
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
422 ih = nil;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
423 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
424 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
425
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
426 if ih then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
427 return setmetatable({ file = ih }, index_mt);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
428 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
429
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
430 local index, err = build_list_index(username, host, datastore);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
431 if not index then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
432 return index, err
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
433 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
434
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
435 -- TODO How to handle failure to store the index?
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
436 local dontcare = store_list_index(username, host, datastore, index); -- luacheck: ignore 211/dontcare
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
437 return index;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
438 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
439
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
440 local function list_load_one(fh, start, length)
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
441 if fh:seek("set", start) ~= start then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
442 return nil
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
443 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
444 local raw_data = fh:read(length)
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
445 if not raw_data or #raw_data ~= length then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
446 return
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
447 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
448 local item;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
449 local data, err, errno = envload(raw_data, "@list", {
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
450 item = function(i)
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
451 item = i;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
452 end;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
453 });
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
454 if not data then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
455 return data, err, errno
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
456 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
457 local success, ret = pcall(data);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
458 if not success then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
459 return success, ret;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
460 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
461 return item;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
462 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
463
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
464 local indexed_list_mt = {
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
465 __index = function(t, i)
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
466 if type(i) ~= "number" or i % 1 ~= 0 or i < 1 then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
467 return
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
468 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
469 local ix = t.index[i];
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
470 if not ix then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
471 return
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
472 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
473 local item = list_load_one(t.file, ix.start, ix.length);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
474 return item;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
475 end;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
476 __len = function(t)
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
477 return #t.index;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
478 end;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
479 }
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
480
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
481 local function list_load(username, host, datastore)
5021
85b2689dbcfe Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents: 4452
diff changeset
482 local items = {};
7925
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
483 local data, err, errno = envloadfile(getpath(username, host, datastore, "list"), {item = function(i) t_insert(items, i); end});
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
484 if not data then
7925
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
485 if errno == ENOENT then
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
486 -- No such file, ok to ignore
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
487 return nil;
209503ee3aaa util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents: 7734
diff changeset
488 end
7996
f4c0fb54e16c util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents: 7995
diff changeset
489 log("error", "Failed to load %s storage ('%s') for user: %s@%s", datastore, err, username or "nil", host or "nil");
f4c0fb54e16c util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents: 7995
diff changeset
490 return nil, "Error reading storage";
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
491 end
5021
85b2689dbcfe Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents: 4452
diff changeset
492
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
493 local success, ret = pcall(data);
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
494 if not success then
4993
5243b74a4cbb Hopefully inert commit to clean up logging across a number of modules, removing all cases of concatenation when building log messages
Matthew Wild <mwild1@gmail.com>
parents: 4452
diff changeset
495 log("error", "Unable to load %s storage ('%s') for user: %s@%s", datastore, ret, username or "nil", host or "nil");
3723
72a917d910a4 util.datamanager: Return an error string when pcall fails on a loaded list file.
Waqas Hussain <waqas20@gmail.com>
parents: 3722
diff changeset
496 return nil, "Error reading storage";
247
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
497 end
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
498 return items;
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
499 end
681b29aa134f Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents: 206
diff changeset
500
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
501 local function list_open(username, host, datastore)
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
502 if not index_magic then
13181
87487056bccb util.datamanager: Reduce log level of left over debug messages to debug (thanks Trung)
Kim Alvefur <zash@zash.se>
parents: 13180
diff changeset
503 log("debug", "Falling back from lazy loading to to loading full list for %s storage for user: %s@%s", datastore, username or "nil", host or "nil");
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
504 return list_load(username, host, datastore);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
505 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
506 local filename = getpath(username, host, datastore, "list");
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
507 local file, err, errno = io_open(filename);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
508 if not file then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
509 if errno == ENOENT then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
510 return nil;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
511 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
512 return file, err, errno;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
513 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
514 local index, err = get_list_index(username, host, datastore);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
515 if not index then
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
516 file:close()
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
517 return index, err;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
518 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
519 return setmetatable({ file = file; index = index }, indexed_list_mt);
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
520 end
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
521
13183
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
522 local function shift_index(index_filename, index, trim_to, offset)
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
523 local index_scratch = index_filename .. "~";
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
524 local new_index, err = io_open(index_scratch, "w");
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
525 if not new_index then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
526 os_remove(index_filename);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
527 return "deleted", err;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
528 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
529
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
530 local ok, err = new_index:write(index_magic);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
531 if not ok then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
532 new_index:close();
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
533 os_remove(index_filename);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
534 os_remove(index_scratch);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
535 return "deleted", err;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
536 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
537
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
538 if not index.file or not index.file:seek("set", index_item_size * trim_to) then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
539 new_index:close();
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
540 os_remove(index_filename);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
541 os_remove(index_scratch);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
542 return "deleted";
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
543 else
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
544 local pack, unpack = string.pack, string.unpack;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
545 for item in index.file:lines(index_item_size) do
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
546 local ok, err = new_index:write(pack(index_fmt, unpack(index_fmt, item) - offset));
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
547 if not ok then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
548 os_remove(index_filename);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
549 os_remove(index_scratch);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
550 return "deleted", err;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
551 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
552 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
553 local ok, err = new_index:close();
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
554 if not ok then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
555 os_remove(index_filename);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
556 os_remove(index_scratch);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
557 return "deleted", err;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
558 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
559 return os_rename(index_scratch, index_filename);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
560 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
561 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
562
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
563 local function list_shift(username, host, datastore, trim_to)
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
564 if trim_to == 1 then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
565 return true
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
566 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
567 if type(trim_to) ~= "number" or trim_to < 1 then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
568 return nil, "invalid-argument";
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
569 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
570 local list_filename = getpath(username, host, datastore, "list");
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
571 local index_filename = getpath(username, host, datastore, "lidx");
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
572 local index, err = get_list_index(username, host, datastore);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
573 if not index then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
574 return nil, err;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
575 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
576
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
577 local new_first = index[trim_to];
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
578 if not new_first then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
579 os_remove(index_filename);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
580 return os_remove(list_filename);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
581 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
582
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
583 local offset = new_first.start;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
584 if offset == 0 then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
585 return true;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
586 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
587
13234
c886ea087b38 util.datamanager: Disable blockwise removal
Kim Alvefur <zash@zash.se>
parents: 13231
diff changeset
588 --[[
13185
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
589 if remove_blocks then
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
590 local f, err = io_open(list_filename, "r+");
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
591 if not f then
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
592 return f, err;
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
593 end
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
594
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
595 local diff = 0;
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
596 local block_offset = 0;
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
597 if offset % 0x1000 ~= 0 then
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
598 -- Not an even block boundary, we will have to overwrite
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
599 diff = offset % 0x1000;
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
600 block_offset = offset - diff;
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
601 end
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
602
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
603 if block_offset == 0 then
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
604 log("debug", "")
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
605 else
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
606 local ok, err = remove_blocks(f, 0, block_offset);
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
607 log("debug", "remove_blocks(%s, 0, %d)", f, block_offset);
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
608 if not ok then
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
609 log("warn", "Could not remove blocks from %q[%d, %d]: %s", list_filename, 0, block_offset, err);
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
610 else
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
611 if diff ~= 0 then
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
612 -- overwrite unaligned leftovers
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
613 if f:seek("set", 0) then
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
614 local wrote, err = f:write(string.rep("\n", diff));
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
615 if not wrote then
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
616 log("error", "Could not blank out %q[%d, %d]: %s", list_filename, 0, diff, err);
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
617 end
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
618 end
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
619 end
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
620 local ok, err = f:close();
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
621 shift_index(index_filename, index, trim_to, offset); -- Shift or delete the index
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
622 return ok, err;
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
623 end
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
624 end
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
625 end
13234
c886ea087b38 util.datamanager: Disable blockwise removal
Kim Alvefur <zash@zash.se>
parents: 13231
diff changeset
626 --]]
13185
b57f45165e1e util.datamanager: Efficiently remove whole blocks to shift lists
Kim Alvefur <zash@zash.se>
parents: 13183
diff changeset
627
13183
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
628 local r, err = io_open(list_filename, "r");
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
629 if not r then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
630 return nil, err;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
631 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
632 local w, err = io_open(list_filename .. "~", "w");
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
633 if not w then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
634 return nil, err;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
635 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
636 r:seek("set", offset);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
637 for block in r:lines(0x1000) do
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
638 local ok, err = w:write(block);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
639 if not ok then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
640 return nil, err;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
641 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
642 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
643 r:close();
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
644 local ok, err = w:close();
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
645 if not ok then
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
646 return nil, err;
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
647 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
648 shift_index(index_filename, index, trim_to, offset)
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
649 return os_rename(list_filename .. "~", list_filename);
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
650 end
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
651
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
652
5130
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
653 local type_map = {
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
654 keyval = "dat";
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
655 list = "list";
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
656 }
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
657
7674
8027eecc750f util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7673
diff changeset
658 local function users(host, store, typ) -- luacheck: ignore 431/store
11369
87105a9a11df util.datamanager: Support iterating over any file extension
Kim Alvefur <zash@zash.se>
parents: 11288
diff changeset
659 typ = "."..(type_map[typ or "keyval"] or typ);
10665
69acda92d5a9 util.datamanager: Fix iterating over "users" (thanks marc0s)
Kim Alvefur <zash@zash.se>
parents: 8555
diff changeset
660 local store_dir = format("%s/%s/%s", data_path, encode(host), store_encode(store));
5153
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
661
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
662 local mode, err = lfs.attributes(store_dir, "mode");
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
663 if not mode then
6644
6cb6855f60df util.datamanager: Fix traceback due to %s in log message
Kim Alvefur <zash@zash.se>
parents: 6575
diff changeset
664 return function() log("debug", "%s", err or (store_dir .. " does not exist")) end
5153
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
665 end
7674
8027eecc750f util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7673
diff changeset
666 local next, state = lfs.dir(store_dir); -- luacheck: ignore 431/next 431/state
8027eecc750f util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7673
diff changeset
667 return function(state) -- luacheck: ignore 431/state
5153
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
668 for node in next, state do
11369
87105a9a11df util.datamanager: Support iterating over any file extension
Kim Alvefur <zash@zash.se>
parents: 11288
diff changeset
669 if node:sub(-#typ, -1) == typ then
87105a9a11df util.datamanager: Support iterating over any file extension
Kim Alvefur <zash@zash.se>
parents: 11288
diff changeset
670 return decode(node:sub(1, -#typ-1));
5153
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
671 end
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
672 end
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
673 end, state;
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
674 end
688aeac0012a mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents: 5130
diff changeset
675
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
676 local function stores(username, host, typ)
5130
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
677 typ = type_map[typ or "keyval"];
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
678 local store_dir = format("%s/%s/", data_path, encode(host));
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
679
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
680 local mode, err = lfs.attributes(store_dir, "mode");
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
681 if not mode then
11288
5fd1f1b544a0 util.datamanager: Fix not passing variables as format string to logger (thanks semgrep)
Kim Alvefur <zash@zash.se>
parents: 10666
diff changeset
682 return function() log("debug", "Could not iterate over stores in %s: %s", store_dir, err); end
5032
c40ea227f8af util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents: 5024
diff changeset
683 end
7674
8027eecc750f util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7673
diff changeset
684 local next, state = lfs.dir(store_dir); -- luacheck: ignore 431/next 431/state
8027eecc750f util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7673
diff changeset
685 return function(state) -- luacheck: ignore 431/state
5130
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
686 for node in next, state do
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
687 if not node:match"^%." then
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
688 if username == true then
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
689 if lfs.attributes(store_dir..node, "mode") == "directory" then
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
690 return decode(node);
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
691 end
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
692 elseif username then
7673
177d569307fd util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7432
diff changeset
693 local store_name = decode(node);
177d569307fd util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7432
diff changeset
694 if lfs.attributes(getpath(username, host, store_name, typ), "mode") then
177d569307fd util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7432
diff changeset
695 return store_name;
5130
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
696 end
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
697 elseif lfs.attributes(node, "mode") == "file" then
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
698 local file, ext = node:match("^(.*)%.([dalist]+)$");
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
699 if ext == typ then
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
700 return decode(file)
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
701 end
5032
c40ea227f8af util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents: 5024
diff changeset
702 end
c40ea227f8af util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents: 5024
diff changeset
703 end
c40ea227f8af util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents: 5024
diff changeset
704 end
5130
051d352ed03c storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents: 5118
diff changeset
705 end, state;
5032
c40ea227f8af util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents: 5024
diff changeset
706 end
c40ea227f8af util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents: 5024
diff changeset
707
5103
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
708 local function do_remove(path)
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
709 local ok, err = os_remove(path);
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
710 if not ok and lfs.attributes(path, "mode") then
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
711 return ok, err;
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
712 end
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
713 return true
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
714 end
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
715
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
716 local function purge(username, host)
5038
242c62ff8e77 util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents: 5032
diff changeset
717 local host_dir = format("%s/%s/", data_path, encode(host));
6681
0217a04722c7 util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents: 6644
diff changeset
718 local ok, iter, state, var = pcall(lfs.dir, host_dir);
0217a04722c7 util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents: 6644
diff changeset
719 if not ok then
0217a04722c7 util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents: 6644
diff changeset
720 return ok, iter;
0217a04722c7 util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents: 6644
diff changeset
721 end
5095
dddbcd62183a util.datamanager: Collect errors when deleting all stores of a user, but ignore "no such file"
Kim Alvefur <zash@zash.se>
parents: 5069
diff changeset
722 local errs = {};
6681
0217a04722c7 util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents: 6644
diff changeset
723 for file in iter, state, var do
5038
242c62ff8e77 util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents: 5032
diff changeset
724 if lfs.attributes(host_dir..file, "mode") == "directory" then
7673
177d569307fd util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7432
diff changeset
725 local store_name = decode(file);
177d569307fd util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7432
diff changeset
726 local ok, err = do_remove(getpath(username, host, store_name));
5103
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
727 if not ok then errs[#errs+1] = err; end
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
728
7673
177d569307fd util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents: 7432
diff changeset
729 local ok, err = do_remove(getpath(username, host, store_name, "list"));
5103
5a1488369c35 util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents: 5095
diff changeset
730 if not ok then errs[#errs+1] = err; end
5038
242c62ff8e77 util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents: 5032
diff changeset
731 end
242c62ff8e77 util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents: 5032
diff changeset
732 end
5095
dddbcd62183a util.datamanager: Collect errors when deleting all stores of a user, but ignore "no such file"
Kim Alvefur <zash@zash.se>
parents: 5069
diff changeset
733 return #errs == 0, t_concat(errs, ", ");
5038
242c62ff8e77 util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents: 5032
diff changeset
734 end
242c62ff8e77 util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents: 5032
diff changeset
735
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
736 return {
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
737 set_data_path = set_data_path;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
738 add_callback = add_callback;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
739 remove_callback = remove_callback;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
740 getpath = getpath;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
741 load = load;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
742 store = store;
7002
9ab0d5e69c41 util.datamanager: Add append to public api
Kim Alvefur <zash@zash.se>
parents: 7001
diff changeset
743 append_raw = append;
7929
4017ea6d2599 util.datamanager: Expose atomic_store as store_raw
Kim Alvefur <zash@zash.se>
parents: 7928
diff changeset
744 store_raw = atomic_store;
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
745 list_append = list_append;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
746 list_store = list_store;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
747 list_load = list_load;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
748 users = users;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
749 stores = stores;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
750 purge = purge;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
751 path_decode = decode;
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
752 path_encode = encode;
13134
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
753
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
754 build_list_index = build_list_index;
638f627e707f util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents: 12975
diff changeset
755 list_open = list_open;
13183
33b114fbb5de util.datamanager: Add way to efficiently remove first items in a list
Kim Alvefur <zash@zash.se>
parents: 13182
diff changeset
756 list_shift = list_shift;
6777
5de6b93d0190 util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents: 6681
diff changeset
757 };