Software /
code /
prosody
Annotate
util/datamanager.lua @ 13180:48622b89f570
util.datamanager: Fix missing separator in log line
author | Kim Alvefur <zash@zash.se> |
---|---|
date | Mon, 10 Jul 2023 17:52:52 +0200 |
parent | 13137:b417a49cc31b |
child | 13181:87487056bccb |
rev | line source |
---|---|
1523
841d61be198f
Remove version number from copyright headers
Matthew Wild <mwild1@gmail.com>
parents:
1462
diff
changeset
|
1 -- Prosody IM |
2923
b7049746bd29
Update copyright headers for 2010
Matthew Wild <mwild1@gmail.com>
parents:
2159
diff
changeset
|
2 -- Copyright (C) 2008-2010 Matthew Wild |
b7049746bd29
Update copyright headers for 2010
Matthew Wild <mwild1@gmail.com>
parents:
2159
diff
changeset
|
3 -- Copyright (C) 2008-2010 Waqas Hussain |
4452
7de17ca4de14
util/datamanager: Use package.config to figure out directory seperator
James Callahan <james@chatid.com>
parents:
4112
diff
changeset
|
4 -- |
758 | 5 -- This project is MIT/X11 licensed. Please see the |
6 -- COPYING file in the source package for more information. | |
519
cccd610a0ef9
Insert copyright/license headers
Matthew Wild <mwild1@gmail.com>
parents:
456
diff
changeset
|
7 -- |
cccd610a0ef9
Insert copyright/license headers
Matthew Wild <mwild1@gmail.com>
parents:
456
diff
changeset
|
8 |
cccd610a0ef9
Insert copyright/license headers
Matthew Wild <mwild1@gmail.com>
parents:
456
diff
changeset
|
9 |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
10 local string = string; |
0 | 11 local format = string.format; |
5051
71253db26fda
util.datamanager: Remove a few unused imports
Kim Alvefur <zash@zash.se>
parents:
5049
diff
changeset
|
12 local setmetatable = setmetatable; |
71253db26fda
util.datamanager: Remove a few unused imports
Kim Alvefur <zash@zash.se>
parents:
5049
diff
changeset
|
13 local ipairs = ipairs; |
0 | 14 local char = string.char; |
5021
85b2689dbcfe
Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents:
4452
diff
changeset
|
15 local pcall = pcall; |
12975
d10957394a3c
util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12388
diff
changeset
|
16 local log = require "prosody.util.logger".init("datamanager"); |
0 | 17 local io_open = io.open; |
206
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
18 local os_remove = os.remove; |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
19 local os_rename = os.rename; |
5051
71253db26fda
util.datamanager: Remove a few unused imports
Kim Alvefur <zash@zash.se>
parents:
5049
diff
changeset
|
20 local tonumber = tonumber; |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
21 local floor = math.floor; |
206
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
22 local next = next; |
7432
92f721226753
util.datamanager: Import tostring and type (fix global access)
Kim Alvefur <zash@zash.se>
parents:
7202
diff
changeset
|
23 local type = type; |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
24 local t_insert = table.insert; |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
25 local t_concat = table.concat; |
12975
d10957394a3c
util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12388
diff
changeset
|
26 local envloadfile = require"prosody.util.envload".envloadfile; |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
27 local envload = require"prosody.util.envload".envload; |
12975
d10957394a3c
util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12388
diff
changeset
|
28 local serialize = require "prosody.util.serialization".serialize; |
3086
931acb1188b1
util.datamanager: When failing to load a file, and the file exists, return nil, error.
Waqas Hussain <waqas20@gmail.com>
parents:
2925
diff
changeset
|
29 local lfs = require "lfs"; |
9993
02a41315d275
Fix various spelling mistakes [codespell]
Kim Alvefur <zash@zash.se>
parents:
8555
diff
changeset
|
30 -- Extract directory separator from package.config (an undocumented string that comes with lua) |
7995
b99404a340fc
util.datamanager: Split long line [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7929
diff
changeset
|
31 local path_separator = assert ( package.config:match ( "^([^\n]+)" ) , "package.config not in standard form" ) |
b99404a340fc
util.datamanager: Split long line [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7929
diff
changeset
|
32 |
4093
36555949bd16
util.datamanager: Use prosody.paths.data as the initial value for data_path
Matthew Wild <mwild1@gmail.com>
parents:
3723
diff
changeset
|
33 local prosody = prosody; |
2444
267d6482bac6
util.datamanager: Use pposix.mkdir if available
Matthew Wild <mwild1@gmail.com>
parents:
2328
diff
changeset
|
34 |
5118
0dc9e6c128c3
util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents:
5103
diff
changeset
|
35 local raw_mkdir = lfs.mkdir; |
8011
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
36 local atomic_append; |
7925
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
37 local ENOENT = 2; |
5118
0dc9e6c128c3
util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents:
5103
diff
changeset
|
38 pcall(function() |
12975
d10957394a3c
util: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12388
diff
changeset
|
39 local pposix = require "prosody.util.pposix"; |
5118
0dc9e6c128c3
util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents:
5103
diff
changeset
|
40 raw_mkdir = pposix.mkdir or raw_mkdir; -- Doesn't trample on umask |
8011
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
41 atomic_append = pposix.atomic_append; |
7927
7132abcf669e
util.datamanager: Import value of ENOENT from util.pposix (if available)
Kim Alvefur <zash@zash.se>
parents:
7925
diff
changeset
|
42 ENOENT = pposix.ENOENT or ENOENT; |
5118
0dc9e6c128c3
util.datamanager: Make the util.pposix dependency optional.
Waqas Hussain <waqas20@gmail.com>
parents:
5103
diff
changeset
|
43 end); |
5045
4ba6940deed0
util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents:
5038
diff
changeset
|
44 |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
45 local _ENV = nil; |
8555
4f0f5b49bb03
vairious: Add annotation when an empty environment is set [luacheck]
Kim Alvefur <zash@zash.se>
parents:
8299
diff
changeset
|
46 -- luacheck: std none |
0 | 47 |
48 ---- utils ----- | |
8299
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
49 local encode, decode, store_encode; |
2444
267d6482bac6
util.datamanager: Use pposix.mkdir if available
Matthew Wild <mwild1@gmail.com>
parents:
2328
diff
changeset
|
50 do |
7732
aba3dd84d9f0
util.datamanager: Use the 'base' argument to tonumber() to indicate hexadecimal
Kim Alvefur <zash@zash.se>
parents:
7674
diff
changeset
|
51 local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber(k, 16)); return t[k]; end }); |
0 | 52 |
53 decode = function (s) | |
7734
6a52415ed68a
util.datamanager: Shorter pattern
Kim Alvefur <zash@zash.se>
parents:
7733
diff
changeset
|
54 return s and (s:gsub("%%(%x%x)", urlcodes)); |
0 | 55 end |
56 | |
57 encode = function (s) | |
628
3712d36b6d25
Fixed URL encoding to generate %0x instead of %x
Waqas Hussain <waqas20@gmail.com>
parents:
615
diff
changeset
|
58 return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); |
0 | 59 end |
8299
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
60 |
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
61 -- Special encode function for store names, which historically were unencoded. |
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
62 -- All currently known stores use a-z and underscore, so this one preserves underscores. |
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
63 store_encode = function (s) |
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
64 return s and (s:gsub("[^_%w]", function (c) return format("%%%02x", c:byte()); end)); |
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
65 end |
0 | 66 end |
67 | |
8011
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
68 if not atomic_append then |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
69 function atomic_append(f, data) |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
70 local pos = f:seek(); |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
71 if not f:write(data) or not f:flush() then |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
72 f:seek("set", pos); |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
73 f:write((" "):rep(#data)); |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
74 f:flush(); |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
75 return nil, "write-failed"; |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
76 end |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
77 return true; |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
78 end |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
79 end |
f8ba814fe029
util.datamanager: Use pposix.atomic_append
Kim Alvefur <zash@zash.se>
parents:
7996
diff
changeset
|
80 |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
81 local _mkdir = {}; |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
82 local function mkdir(path) |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
83 path = path:gsub("/", path_separator); -- TODO as an optimization, do this during path creation rather than here |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
84 if not _mkdir[path] then |
2444
267d6482bac6
util.datamanager: Use pposix.mkdir if available
Matthew Wild <mwild1@gmail.com>
parents:
2328
diff
changeset
|
85 raw_mkdir(path); |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
86 _mkdir[path] = true; |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
87 end |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
88 return path; |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
89 end |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
90 |
4108
e3e3aa286334
util.datamanager: Handle gracefully the lack of prosody.paths.data
Matthew Wild <mwild1@gmail.com>
parents:
4093
diff
changeset
|
91 local data_path = (prosody and prosody.paths and prosody.paths.data) or "."; |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
92 local callbacks = {}; |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
93 |
0 | 94 ------- API ------------- |
95 | |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
96 local function set_data_path(path) |
1097
c5b33640a5f0
util.datamanager: Lower log level of 'Setting data path' to 'debug'
Matthew Wild <mwild1@gmail.com>
parents:
932
diff
changeset
|
97 log("debug", "Setting data path to: %s", path); |
452
613c5c6bdce4
Added option core.data_path
Waqas Hussain <waqas20@gmail.com>
parents:
267
diff
changeset
|
98 data_path = path; |
613c5c6bdce4
Added option core.data_path
Waqas Hussain <waqas20@gmail.com>
parents:
267
diff
changeset
|
99 end |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
100 |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
101 local function callback(username, host, datastore, data) |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
102 for _, f in ipairs(callbacks) do |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
103 username, host, datastore, data = f(username, host, datastore, data); |
1462
44780b856ce7
datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents:
1381
diff
changeset
|
104 if username == false then break; end |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
105 end |
4452
7de17ca4de14
util/datamanager: Use package.config to figure out directory seperator
James Callahan <james@chatid.com>
parents:
4112
diff
changeset
|
106 |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
107 return username, host, datastore, data; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
108 end |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
109 local function add_callback(func) |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
110 if not callbacks[func] then -- Would you really want to set the same callback more than once? |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
111 callbacks[func] = true; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
112 callbacks[#callbacks+1] = func; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
113 return true; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
114 end |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
115 end |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
116 local function remove_callback(func) |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
117 if callbacks[func] then |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
118 for i, f in ipairs(callbacks) do |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
119 if f == func then |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
120 callbacks[i] = nil; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
121 callbacks[f] = nil; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
122 return true; |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
123 end |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
124 end |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
125 end |
932
1ed3e5fe165a
Added: datamanager: Allow a callback to be installed which selectively prevents disk writes
Waqas Hussain <waqas20@gmail.com>
parents:
915
diff
changeset
|
126 end |
452
613c5c6bdce4
Added option core.data_path
Waqas Hussain <waqas20@gmail.com>
parents:
267
diff
changeset
|
127 |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
128 local function getpath(username, host, datastore, ext, create) |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
129 ext = ext or "dat"; |
2328
2804d81206d8
util.datamanager: Store data stores with no host in '_global' folder
Matthew Wild <mwild1@gmail.com>
parents:
2159
diff
changeset
|
130 host = (host and encode(host)) or "_global"; |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
131 username = username and encode(username); |
8299
756a2a00e7e7
util.datamanager: Encode the 'store' path component, preserving underscores
Kim Alvefur <zash@zash.se>
parents:
8092
diff
changeset
|
132 datastore = store_encode(datastore); |
84
d0a0bac6815e
Added: Datastore support for hosts and global data in addition to users
Waqas Hussain <waqas20@gmail.com>
parents:
0
diff
changeset
|
133 if username then |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
134 if create then mkdir(mkdir(mkdir(data_path).."/"..host).."/"..datastore); end |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
135 return format("%s/%s/%s/%s.%s", data_path, host, datastore, username, ext); |
5243
07e8256efcda
util.datamanager: Remove dead code path
Kim Alvefur <zash@zash.se>
parents:
5153
diff
changeset
|
136 else |
643
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
137 if create then mkdir(mkdir(data_path).."/"..host); end |
8ff454831f7d
Moved directory auto-creation to datamanager
Waqas Hussain <waqas20@gmail.com>
parents:
628
diff
changeset
|
138 return format("%s/%s/%s.%s", data_path, host, datastore, ext); |
84
d0a0bac6815e
Added: Datastore support for hosts and global data in addition to users
Waqas Hussain <waqas20@gmail.com>
parents:
0
diff
changeset
|
139 end |
0 | 140 end |
141 | |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
142 local function load(username, host, datastore) |
7925
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
143 local data, err, errno = envloadfile(getpath(username, host, datastore), {}); |
117
8e5c5e6a3240
Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents:
88
diff
changeset
|
144 if not data then |
7925
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
145 if errno == ENOENT then |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
146 -- No such file, ok to ignore |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
147 return nil; |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
148 end |
7996
f4c0fb54e16c
util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents:
7995
diff
changeset
|
149 log("error", "Failed to load %s storage ('%s') for user: %s@%s", datastore, err, username or "nil", host or "nil"); |
f4c0fb54e16c
util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents:
7995
diff
changeset
|
150 return nil, "Error reading storage"; |
117
8e5c5e6a3240
Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents:
88
diff
changeset
|
151 end |
5021
85b2689dbcfe
Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents:
4452
diff
changeset
|
152 |
0 | 153 local success, ret = pcall(data); |
117
8e5c5e6a3240
Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents:
88
diff
changeset
|
154 if not success then |
4993
5243b74a4cbb
Hopefully inert commit to clean up logging across a number of modules, removing all cases of concatenation when building log messages
Matthew Wild <mwild1@gmail.com>
parents:
4452
diff
changeset
|
155 log("error", "Unable to load %s storage ('%s') for user: %s@%s", datastore, ret, username or "nil", host or "nil"); |
3086
931acb1188b1
util.datamanager: When failing to load a file, and the file exists, return nil, error.
Waqas Hussain <waqas20@gmail.com>
parents:
2925
diff
changeset
|
156 return nil, "Error reading storage"; |
117
8e5c5e6a3240
Fixed: datamanager.store and datamanager.load could crash when username or host arguments were nil. (useful for server specific and global data).
Waqas Hussain <waqas20@gmail.com>
parents:
88
diff
changeset
|
157 end |
0 | 158 return ret; |
159 end | |
160 | |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
161 local function atomic_store(filename, data) |
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
162 local scratch = filename.."~"; |
10531
810abf34919a
util.datamanager: Ignore unused 'errno' variable [luacheck]
Kim Alvefur <zash@zash.se>
parents:
9993
diff
changeset
|
163 local f, ok, msg, errno; -- luacheck: ignore errno |
810abf34919a
util.datamanager: Ignore unused 'errno' variable [luacheck]
Kim Alvefur <zash@zash.se>
parents:
9993
diff
changeset
|
164 -- TODO return util.error with code=errno? |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
165 |
7925
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
166 f, msg, errno = io_open(scratch, "w"); |
7202
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
167 if not f then |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
168 return nil, msg; |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
169 end |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
170 |
7202
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
171 ok, msg = f:write(data); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
172 if not ok then |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
173 f:close(); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
174 os_remove(scratch); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
175 return nil, msg; |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
176 end |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
177 |
7202
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
178 ok, msg = f:close(); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
179 if not ok then |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
180 os_remove(scratch); |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
181 return nil, msg; |
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
182 end |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
183 |
7202
5bf0ff3882aa
util.datamanager: Explicit handling of each error condition (see #632)
Kim Alvefur <zash@zash.se>
parents:
7201
diff
changeset
|
184 return os_rename(scratch, filename); |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
185 end |
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
186 |
6575
bdaadf70a48f
util.datamanager: Check that the global 'prosody' exists before using it (fixes nil indexing in use outside of prosody)
Kim Alvefur <zash@zash.se>
parents:
5441
diff
changeset
|
187 if prosody and prosody.platform ~= "posix" then |
5065
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
188 -- os.rename does not overwrite existing files on Windows |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
189 -- TODO We could use Transactional NTFS on Vista and above |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
190 function atomic_store(filename, data) |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
191 local f, err = io_open(filename, "w"); |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
192 if not f then return f, err; end |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
193 local ok, msg = f:write(data); |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
194 if not ok then f:close(); return ok, msg; end |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
195 return f:close(); |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
196 end |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
197 end |
acfaf771f10e
util.datamanager: Don't use os.rename on non-POSIX. It doesn't overwrite exisitng files on Windows.
Waqas Hussain <waqas20@gmail.com>
parents:
5057
diff
changeset
|
198 |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
199 local function store(username, host, datastore, data) |
206
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
200 if not data then |
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
201 data = {}; |
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
202 end |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
203 |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
204 username, host, datastore, data = callback(username, host, datastore, data); |
1462
44780b856ce7
datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents:
1381
diff
changeset
|
205 if username == false then |
1381
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
206 return true; -- Don't save this data at all |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
207 end |
46a58df8557d
util.datamanager: Allow multiple data storage callbacks, and allow them to modify parameters
Matthew Wild <mwild1@gmail.com>
parents:
1097
diff
changeset
|
208 |
206
e30d0e30a0ff
Datamanager now deletes files with no data
Waqas Hussain <waqas20@gmail.com>
parents:
182
diff
changeset
|
209 -- save the datastore |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
210 local d = "return " .. serialize(data) .. ";\n"; |
5441
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
211 local mkdir_cache_cleared; |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
212 repeat |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
213 local ok, msg = atomic_store(getpath(username, host, datastore, nil, true), d); |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
214 if not ok then |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
215 if not mkdir_cache_cleared then -- We may need to recreate a removed directory |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
216 _mkdir = {}; |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
217 mkdir_cache_cleared = true; |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
218 else |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
219 log("error", "Unable to write to %s storage ('%s') for user: %s@%s", datastore, msg, username or "nil", host or "nil"); |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
220 return nil, "Error saving to storage"; |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
221 end |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
222 end |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
223 if next(data) == nil then -- try to delete empty datastore |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
224 log("debug", "Removing empty %s datastore for user %s@%s", datastore, username or "nil", host or "nil"); |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
225 os_remove(getpath(username, host, datastore)); |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
226 end |
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
227 -- we write data even when we are deleting because lua doesn't have a |
12388
50fcd3879482
spelling: non-existing mistakes (thanks timeless)
Kim Alvefur <zash@zash.se>
parents:
12387
diff
changeset
|
228 -- platform independent way of checking for nonexisting files |
5441
6a5c622cc6d4
util.datamanager: Clear the cache of created directories on storage failure, and retry
Matthew Wild <mwild1@gmail.com>
parents:
5440
diff
changeset
|
229 until ok; |
0 | 230 return true; |
231 end | |
232 | |
6999
0ad66d12113a
util.datamanager: Add some comments about the append function
Kim Alvefur <zash@zash.se>
parents:
6998
diff
changeset
|
233 -- Append a blob of data to a file |
6992
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6777
diff
changeset
|
234 local function append(username, host, datastore, ext, data) |
7000
2b57f77985a3
util.datamanager: Make sure only strings are passed as data to append()
Kim Alvefur <zash@zash.se>
parents:
6999
diff
changeset
|
235 if type(data) ~= "string" then return; end |
6994
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6993
diff
changeset
|
236 local filename = getpath(username, host, datastore, ext, true); |
6996
644b1bddc676
util.datamanager: No shadowing of variable [luacheck]
Kim Alvefur <zash@zash.se>
parents:
6995
diff
changeset
|
237 |
8013
72cfbe377326
util.datamanager: Rearrange locals
Kim Alvefur <zash@zash.se>
parents:
8011
diff
changeset
|
238 local f = io_open(filename, "r+"); |
5066
4be7093edde9
util.datamanager: Try to open in read+write mode, then retry with write mode if that fails (usually because it doesn't exist)
Kim Alvefur <zash@zash.se>
parents:
5057
diff
changeset
|
239 if not f then |
7928
9d014aca766f
util.datamanager: Use atomic store function when writing the first entry of a list
Kim Alvefur <zash@zash.se>
parents:
7927
diff
changeset
|
240 return atomic_store(filename, data); |
6999
0ad66d12113a
util.datamanager: Add some comments about the append function
Kim Alvefur <zash@zash.se>
parents:
6998
diff
changeset
|
241 -- File did probably not exist, let's create it |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
242 end |
6999
0ad66d12113a
util.datamanager: Add some comments about the append function
Kim Alvefur <zash@zash.se>
parents:
6998
diff
changeset
|
243 |
5045
4ba6940deed0
util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents:
5038
diff
changeset
|
244 local pos = f:seek("end"); |
6997
0ab228bc21c6
util.datamanager: Handle potential issues from fallocate
Kim Alvefur <zash@zash.se>
parents:
6996
diff
changeset
|
245 |
8013
72cfbe377326
util.datamanager: Rearrange locals
Kim Alvefur <zash@zash.se>
parents:
8011
diff
changeset
|
246 local ok, msg = atomic_append(f, data); |
6997
0ab228bc21c6
util.datamanager: Handle potential issues from fallocate
Kim Alvefur <zash@zash.se>
parents:
6996
diff
changeset
|
247 |
6998
86607fe755b6
util.datamanager: Handle potential error from :write() call
Kim Alvefur <zash@zash.se>
parents:
6997
diff
changeset
|
248 if not ok then |
86607fe755b6
util.datamanager: Handle potential error from :write() call
Kim Alvefur <zash@zash.se>
parents:
6997
diff
changeset
|
249 f:close(); |
7001
2743759ca1b5
util.datamanager: Return extra location info
Kim Alvefur <zash@zash.se>
parents:
7000
diff
changeset
|
250 return ok, msg, "write"; |
6992
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6777
diff
changeset
|
251 end |
6994
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6993
diff
changeset
|
252 |
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6993
diff
changeset
|
253 ok, msg = f:close(); |
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6993
diff
changeset
|
254 if not ok then |
8014
ff3787033abb
util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents:
8013
diff
changeset
|
255 return ok, msg, "close"; |
6994
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6993
diff
changeset
|
256 end |
507301531cf5
util.datamanager: In append() collect status when closing file handle as it may fail (eg the implied flush)
Kim Alvefur <zash@zash.se>
parents:
6993
diff
changeset
|
257 |
7001
2743759ca1b5
util.datamanager: Return extra location info
Kim Alvefur <zash@zash.se>
parents:
7000
diff
changeset
|
258 return true, pos; |
6992
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6777
diff
changeset
|
259 end |
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6777
diff
changeset
|
260 |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
261 local index_fmt, index_item_size, index_magic; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
262 if string.packsize then |
13137
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
263 index_fmt = "T"; -- offset to the end of the item, length can be derived from two index items |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
264 index_item_size = string.packsize(index_fmt); |
13137
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
265 index_magic = string.pack(index_fmt, 7767639 + 1); -- Magic string: T9 for "prosody", version number |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
266 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
267 |
6992
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6777
diff
changeset
|
268 local function list_append(username, host, datastore, data) |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
269 if not data then return; end |
1462
44780b856ce7
datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents:
1381
diff
changeset
|
270 if callback(username, host, datastore) == false then return true; end |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
271 -- save the datastore |
6992
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6777
diff
changeset
|
272 |
6993
dc0c6b8dc638
util.datamanager: Overwrite 'data' variable instead of shadownig it [luacheck]
Kim Alvefur <zash@zash.se>
parents:
6992
diff
changeset
|
273 data = "item(" .. serialize(data) .. ");\n"; |
8014
ff3787033abb
util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents:
8013
diff
changeset
|
274 local ok, msg, where = append(username, host, datastore, "list", data); |
6992
0622f2820d1d
util.datamanager: Factor out code for appending bytes to a file
Kim Alvefur <zash@zash.se>
parents:
6777
diff
changeset
|
275 if not ok then |
8014
ff3787033abb
util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents:
8013
diff
changeset
|
276 log("error", "Unable to write to %s storage ('%s' in %s) for user: %s@%s", |
ff3787033abb
util.datamanager: Log where the error came from
Kim Alvefur <zash@zash.se>
parents:
8013
diff
changeset
|
277 datastore, msg, where, username or "nil", host or "nil"); |
5045
4ba6940deed0
util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents:
5038
diff
changeset
|
278 return ok, msg; |
4ba6940deed0
util.datamanager: Use pposix.fallocate() to make sure appends succeed. Also add a fallback fallocate()
Kim Alvefur <zash@zash.se>
parents:
5038
diff
changeset
|
279 end |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
280 if string.packsize then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
281 local offset = type(msg) == "number" and msg or 0; |
13137
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
282 local index_entry = string.pack(index_fmt, offset + #data); |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
283 if offset == 0 then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
284 index_entry = index_magic .. index_entry; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
285 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
286 local ok, off = append(username, host, datastore, "lidx", index_entry); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
287 off = off or 0; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
288 -- If this was the first item, then both the data and index offsets should |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
289 -- be zero, otherwise there's some kind of mismatch and we should drop the |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
290 -- index and recreate it from scratch |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
291 -- TODO Actually rebuild the index in this case? |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
292 if not ok or (off == 0 and offset ~= 0) or (off ~= 0 and offset == 0) then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
293 os_remove(getpath(username, host, datastore, "lidx")); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
294 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
295 end |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
296 return true; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
297 end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
298 |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
299 local function list_store(username, host, datastore, data) |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
300 if not data then |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
301 data = {}; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
302 end |
1462
44780b856ce7
datamanager: Fixed incorrect callback result checking
Waqas Hussain <waqas20@gmail.com>
parents:
1381
diff
changeset
|
303 if callback(username, host, datastore) == false then return true; end |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
304 -- save the datastore |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
305 local d = {}; |
8092
0a1c0f1107d2
util.datamanager: Use already known index instead of measuring length each iteration
Kim Alvefur <zash@zash.se>
parents:
8014
diff
changeset
|
306 for i, item in ipairs(data) do |
0a1c0f1107d2
util.datamanager: Use already known index instead of measuring length each iteration
Kim Alvefur <zash@zash.se>
parents:
8014
diff
changeset
|
307 d[i] = "item(" .. serialize(item) .. ");\n"; |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
308 end |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
309 os_remove(getpath(username, host, datastore, "lidx")); |
5049
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
310 local ok, msg = atomic_store(getpath(username, host, datastore, "list", true), t_concat(d)); |
5d685f123332
util.datamanager: Write to a temporary file and atomically move it into place
Kim Alvefur <zash@zash.se>
parents:
5045
diff
changeset
|
311 if not ok then |
4993
5243b74a4cbb
Hopefully inert commit to clean up logging across a number of modules, removing all cases of concatenation when building log messages
Matthew Wild <mwild1@gmail.com>
parents:
4452
diff
changeset
|
312 log("error", "Unable to write to %s storage ('%s') for user: %s@%s", datastore, msg, username or "nil", host or "nil"); |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
313 return; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
314 end |
915
0fe5bf7ab81d
util.datamanager: Don't delete data when first entry in table is 'false'. My favourite bug so far.
Matthew Wild <mwild1@gmail.com>
parents:
896
diff
changeset
|
315 if next(data) == nil then -- try to delete empty datastore |
1732
f1282fad2f99
datamanager: Fixed logging errors on deletion of datastores not owned by a user@host
Waqas Hussain <waqas20@gmail.com>
parents:
1523
diff
changeset
|
316 log("debug", "Removing empty %s datastore for user %s@%s", datastore, username or "nil", host or "nil"); |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
317 os_remove(getpath(username, host, datastore, "list")); |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
318 end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
319 -- we write data even when we are deleting because lua doesn't have a |
12388
50fcd3879482
spelling: non-existing mistakes (thanks timeless)
Kim Alvefur <zash@zash.se>
parents:
12387
diff
changeset
|
320 -- platform independent way of checking for nonexisting files |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
321 return true; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
322 end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
323 |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
324 local function build_list_index(username, host, datastore, items) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
325 log("debug", "Building index for (%s@%s/%s)", username, host, datastore); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
326 local filename = getpath(username, host, datastore, "list"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
327 local fh, err, errno = io_open(filename); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
328 if not fh then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
329 return fh, err, errno; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
330 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
331 local prev_pos = 0; -- position before reading |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
332 local last_item_start = 0; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
333 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
334 if items and items[1] then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
335 local last_item = items[#items]; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
336 last_item_start = fh:seek("set", last_item.start + last_item.length); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
337 else |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
338 items = {}; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
339 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
340 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
341 for line in fh:lines() do |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
342 if line:sub(1, 4) == "item" then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
343 if prev_pos ~= 0 then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
344 t_insert(items, { start = last_item_start; length = prev_pos - last_item_start }); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
345 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
346 last_item_start = prev_pos |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
347 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
348 -- seek position is at the start of the next line within each loop iteration |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
349 -- so we need to collect the "current" position at the end of the previous |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
350 prev_pos = fh:seek() |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
351 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
352 if prev_pos ~= 0 then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
353 t_insert(items, { start = last_item_start; length = prev_pos - last_item_start }); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
354 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
355 return items; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
356 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
357 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
358 local function store_list_index(username, host, datastore, index) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
359 local data = { index_magic }; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
360 for i, v in ipairs(index) do |
13137
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
361 data[i + 1] = string.pack(index_fmt, v.start + v.length); |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
362 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
363 local filename = getpath(username, host, datastore, "lidx"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
364 return atomic_store(filename, t_concat(data)); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
365 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
366 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
367 local index_mt = { |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
368 __index = function(t, i) |
13137
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
369 if type(i) ~= "number" or i % 1 ~= 0 or i < 0 then |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
370 return |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
371 end |
13137
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
372 if i <= 0 then |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
373 return 0 |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
374 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
375 local fh = t.file; |
13137
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
376 local pos = (i - 1) * index_item_size; |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
377 if fh:seek("set", pos) ~= pos then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
378 return nil |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
379 end |
13137
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
380 local data = fh:read(index_item_size * 2); |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
381 if not data or #data ~= index_item_size * 2 then |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
382 return nil |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
383 end |
13137
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
384 local start, next_pos = string.unpack(index_fmt .. index_fmt, data); |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
385 if pos == 0 then |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
386 start = 0 |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
387 end |
b417a49cc31b
util.datamanager: Halve size of list index
Kim Alvefur <zash@zash.se>
parents:
13134
diff
changeset
|
388 local length = next_pos - start; |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
389 local v = { start = start; length = length }; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
390 t[i] = v; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
391 return v; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
392 end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
393 __len = function(t) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
394 -- Account for both the header and the fence post error |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
395 return floor(t.file:seek("end") / index_item_size) - 1; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
396 end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
397 } |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
398 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
399 local function get_list_index(username, host, datastore) |
13180
48622b89f570
util.datamanager: Fix missing separator in log line
Kim Alvefur <zash@zash.se>
parents:
13137
diff
changeset
|
400 log("debug", "Loading index for (%s@%s/%s)", username, host, datastore); |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
401 local index_filename = getpath(username, host, datastore, "lidx"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
402 local ih = io_open(index_filename); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
403 if ih then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
404 local magic = ih:read(#index_magic); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
405 if magic ~= index_magic then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
406 log("warn", "Index %q has wrong version number (got %q, expected %q)", index_filename, magic, index_magic); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
407 -- wrong version or something |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
408 ih:close(); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
409 ih = nil; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
410 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
411 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
412 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
413 if ih then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
414 return setmetatable({ file = ih }, index_mt); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
415 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
416 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
417 local index, err = build_list_index(username, host, datastore); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
418 if not index then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
419 return index, err |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
420 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
421 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
422 -- TODO How to handle failure to store the index? |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
423 local dontcare = store_list_index(username, host, datastore, index); -- luacheck: ignore 211/dontcare |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
424 return index; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
425 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
426 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
427 local function list_load_one(fh, start, length) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
428 if fh:seek("set", start) ~= start then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
429 return nil |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
430 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
431 local raw_data = fh:read(length) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
432 if not raw_data or #raw_data ~= length then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
433 return |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
434 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
435 local item; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
436 local data, err, errno = envload(raw_data, "@list", { |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
437 item = function(i) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
438 item = i; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
439 end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
440 }); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
441 if not data then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
442 return data, err, errno |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
443 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
444 local success, ret = pcall(data); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
445 if not success then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
446 return success, ret; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
447 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
448 return item; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
449 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
450 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
451 local indexed_list_mt = { |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
452 __index = function(t, i) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
453 if type(i) ~= "number" or i % 1 ~= 0 or i < 1 then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
454 return |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
455 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
456 local ix = t.index[i]; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
457 if not ix then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
458 return |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
459 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
460 local item = list_load_one(t.file, ix.start, ix.length); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
461 return item; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
462 end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
463 __len = function(t) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
464 return #t.index; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
465 end; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
466 } |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
467 |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
468 local function list_load(username, host, datastore) |
5021
85b2689dbcfe
Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents:
4452
diff
changeset
|
469 local items = {}; |
7925
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
470 local data, err, errno = envloadfile(getpath(username, host, datastore, "list"), {item = function(i) t_insert(items, i); end}); |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
471 if not data then |
7925
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
472 if errno == ENOENT then |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
473 -- No such file, ok to ignore |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
474 return nil; |
209503ee3aaa
util.datamanager: Ignore ENOENT (no such file) when loading data
Kim Alvefur <zash@zash.se>
parents:
7734
diff
changeset
|
475 end |
7996
f4c0fb54e16c
util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents:
7995
diff
changeset
|
476 log("error", "Failed to load %s storage ('%s') for user: %s@%s", datastore, err, username or "nil", host or "nil"); |
f4c0fb54e16c
util.datamanager: Remove check for file non-existence since ENOENT tells us that already
Kim Alvefur <zash@zash.se>
parents:
7995
diff
changeset
|
477 return nil, "Error reading storage"; |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
478 end |
5021
85b2689dbcfe
Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents:
4452
diff
changeset
|
479 |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
480 local success, ret = pcall(data); |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
481 if not success then |
4993
5243b74a4cbb
Hopefully inert commit to clean up logging across a number of modules, removing all cases of concatenation when building log messages
Matthew Wild <mwild1@gmail.com>
parents:
4452
diff
changeset
|
482 log("error", "Unable to load %s storage ('%s') for user: %s@%s", datastore, ret, username or "nil", host or "nil"); |
3723
72a917d910a4
util.datamanager: Return an error string when pcall fails on a loaded list file.
Waqas Hussain <waqas20@gmail.com>
parents:
3722
diff
changeset
|
483 return nil, "Error reading storage"; |
247
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
484 end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
485 return items; |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
486 end |
681b29aa134f
Added support for storing (and removing), loading and appending to lists of data to datamanager (for supporting offline messages)
Waqas Hussain <waqas20@gmail.com>
parents:
206
diff
changeset
|
487 |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
488 local function list_open(username, host, datastore) |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
489 if not index_magic then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
490 log("warn", "Falling back from lazy loading to to loading full list for %s storage for user: %s@%s", datastore, username or "nil", host or "nil"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
491 return list_load(username, host, datastore); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
492 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
493 local filename = getpath(username, host, datastore, "list"); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
494 local file, err, errno = io_open(filename); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
495 if not file then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
496 if errno == ENOENT then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
497 return nil; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
498 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
499 return file, err, errno; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
500 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
501 local index, err = get_list_index(username, host, datastore); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
502 if not index then |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
503 file:close() |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
504 return index, err; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
505 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
506 return setmetatable({ file = file; index = index }, indexed_list_mt); |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
507 end |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
508 |
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
509 local type_map = { |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
510 keyval = "dat"; |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
511 list = "list"; |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
512 } |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
513 |
7674
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7673
diff
changeset
|
514 local function users(host, store, typ) -- luacheck: ignore 431/store |
11369
87105a9a11df
util.datamanager: Support iterating over any file extension
Kim Alvefur <zash@zash.se>
parents:
11288
diff
changeset
|
515 typ = "."..(type_map[typ or "keyval"] or typ); |
10665
69acda92d5a9
util.datamanager: Fix iterating over "users" (thanks marc0s)
Kim Alvefur <zash@zash.se>
parents:
8555
diff
changeset
|
516 local store_dir = format("%s/%s/%s", data_path, encode(host), store_encode(store)); |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
517 |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
518 local mode, err = lfs.attributes(store_dir, "mode"); |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
519 if not mode then |
6644
6cb6855f60df
util.datamanager: Fix traceback due to %s in log message
Kim Alvefur <zash@zash.se>
parents:
6575
diff
changeset
|
520 return function() log("debug", "%s", err or (store_dir .. " does not exist")) end |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
521 end |
7674
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7673
diff
changeset
|
522 local next, state = lfs.dir(store_dir); -- luacheck: ignore 431/next 431/state |
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7673
diff
changeset
|
523 return function(state) -- luacheck: ignore 431/state |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
524 for node in next, state do |
11369
87105a9a11df
util.datamanager: Support iterating over any file extension
Kim Alvefur <zash@zash.se>
parents:
11288
diff
changeset
|
525 if node:sub(-#typ, -1) == typ then |
87105a9a11df
util.datamanager: Support iterating over any file extension
Kim Alvefur <zash@zash.se>
parents:
11288
diff
changeset
|
526 return decode(node:sub(1, -#typ-1)); |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
527 end |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
528 end |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
529 end, state; |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
530 end |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5130
diff
changeset
|
531 |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
532 local function stores(username, host, typ) |
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
533 typ = type_map[typ or "keyval"]; |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
534 local store_dir = format("%s/%s/", data_path, encode(host)); |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
535 |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
536 local mode, err = lfs.attributes(store_dir, "mode"); |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
537 if not mode then |
11288
5fd1f1b544a0
util.datamanager: Fix not passing variables as format string to logger (thanks semgrep)
Kim Alvefur <zash@zash.se>
parents:
10666
diff
changeset
|
538 return function() log("debug", "Could not iterate over stores in %s: %s", store_dir, err); end |
5032
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
539 end |
7674
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7673
diff
changeset
|
540 local next, state = lfs.dir(store_dir); -- luacheck: ignore 431/next 431/state |
8027eecc750f
util.datamanager: Add annotations to ignore name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7673
diff
changeset
|
541 return function(state) -- luacheck: ignore 431/state |
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
542 for node in next, state do |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
543 if not node:match"^%." then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
544 if username == true then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
545 if lfs.attributes(store_dir..node, "mode") == "directory" then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
546 return decode(node); |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
547 end |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
548 elseif username then |
7673
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7432
diff
changeset
|
549 local store_name = decode(node); |
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7432
diff
changeset
|
550 if lfs.attributes(getpath(username, host, store_name, typ), "mode") then |
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7432
diff
changeset
|
551 return store_name; |
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
552 end |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
553 elseif lfs.attributes(node, "mode") == "file" then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
554 local file, ext = node:match("^(.*)%.([dalist]+)$"); |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
555 if ext == typ then |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
556 return decode(file) |
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
557 end |
5032
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
558 end |
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
559 end |
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
560 end |
5130
051d352ed03c
storagemanager, datamanager, mod_storage_{internal,sql}: Replace list_stores() with an iterator version
Kim Alvefur <zash@zash.se>
parents:
5118
diff
changeset
|
561 end, state; |
5032
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
562 end |
c40ea227f8af
util.datamanager: Add function for listing stores
Kim Alvefur <zash@zash.se>
parents:
5024
diff
changeset
|
563 |
5103
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
564 local function do_remove(path) |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
565 local ok, err = os_remove(path); |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
566 if not ok and lfs.attributes(path, "mode") then |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
567 return ok, err; |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
568 end |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
569 return true |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
570 end |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
571 |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
572 local function purge(username, host) |
5038
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
573 local host_dir = format("%s/%s/", data_path, encode(host)); |
6681
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6644
diff
changeset
|
574 local ok, iter, state, var = pcall(lfs.dir, host_dir); |
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6644
diff
changeset
|
575 if not ok then |
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6644
diff
changeset
|
576 return ok, iter; |
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6644
diff
changeset
|
577 end |
5095
dddbcd62183a
util.datamanager: Collect errors when deleting all stores of a user, but ignore "no such file"
Kim Alvefur <zash@zash.se>
parents:
5069
diff
changeset
|
578 local errs = {}; |
6681
0217a04722c7
util.datamanager: Fix traceback from trying to purge when storage is empty or otherwise unaccessible (fixes #496)
Kim Alvefur <zash@zash.se>
parents:
6644
diff
changeset
|
579 for file in iter, state, var do |
5038
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
580 if lfs.attributes(host_dir..file, "mode") == "directory" then |
7673
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7432
diff
changeset
|
581 local store_name = decode(file); |
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7432
diff
changeset
|
582 local ok, err = do_remove(getpath(username, host, store_name)); |
5103
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
583 if not ok then errs[#errs+1] = err; end |
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
584 |
7673
177d569307fd
util.datamanager: Rename variables to avoid name clashes [luacheck]
Kim Alvefur <zash@zash.se>
parents:
7432
diff
changeset
|
585 local ok, err = do_remove(getpath(username, host, store_name, "list")); |
5103
5a1488369c35
util.datamanager: Ignore errors if the file is gone after removing it
Kim Alvefur <zash@zash.se>
parents:
5095
diff
changeset
|
586 if not ok then errs[#errs+1] = err; end |
5038
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
587 end |
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
588 end |
5095
dddbcd62183a
util.datamanager: Collect errors when deleting all stores of a user, but ignore "no such file"
Kim Alvefur <zash@zash.se>
parents:
5069
diff
changeset
|
589 return #errs == 0, t_concat(errs, ", "); |
5038
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
590 end |
242c62ff8e77
util.datamanager: Add function for removing all data belonging to a user
Kim Alvefur <zash@zash.se>
parents:
5032
diff
changeset
|
591 |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
592 return { |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
593 set_data_path = set_data_path; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
594 add_callback = add_callback; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
595 remove_callback = remove_callback; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
596 getpath = getpath; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
597 load = load; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
598 store = store; |
7002
9ab0d5e69c41
util.datamanager: Add append to public api
Kim Alvefur <zash@zash.se>
parents:
7001
diff
changeset
|
599 append_raw = append; |
7929
4017ea6d2599
util.datamanager: Expose atomic_store as store_raw
Kim Alvefur <zash@zash.se>
parents:
7928
diff
changeset
|
600 store_raw = atomic_store; |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
601 list_append = list_append; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
602 list_store = list_store; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
603 list_load = list_load; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
604 users = users; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
605 stores = stores; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
606 purge = purge; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
607 path_decode = decode; |
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
608 path_encode = encode; |
13134
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
609 |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
610 build_list_index = build_list_index; |
638f627e707f
util.datamanager: Add O(1) list indexing with on-disk index
Kim Alvefur <zash@zash.se>
parents:
12975
diff
changeset
|
611 list_open = list_open; |
6777
5de6b93d0190
util.*: Remove use of module() function, make all module functions local and return them in a table at the end
Kim Alvefur <zash@zash.se>
parents:
6681
diff
changeset
|
612 }; |