Software /
code /
prosody
Annotate
plugins/mod_storage_internal.lua @ 13228:616c578c644f
mod_storage_internal: Use integer option method for cache size
Missed this one in previous sweep
author | Kim Alvefur <zash@zash.se> |
---|---|
date | Tue, 18 Jul 2023 12:36:47 +0200 |
parent | 13213:50324f66ca2a |
child | 13262:9a86e7cbdd79 |
rev | line source |
---|---|
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
11765
diff
changeset
|
1 local cache = require "prosody.util.cache"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
11765
diff
changeset
|
2 local datamanager = require "prosody.core.storagemanager".olddm; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
11765
diff
changeset
|
3 local array = require "prosody.util.array"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
11765
diff
changeset
|
4 local datetime = require "prosody.util.datetime"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
11765
diff
changeset
|
5 local st = require "prosody.util.stanza"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
11765
diff
changeset
|
6 local now = require "prosody.util.time".now; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
11765
diff
changeset
|
7 local id = require "prosody.util.id".medium; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
11765
diff
changeset
|
8 local jid_join = require "prosody.util.jid".join; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
11765
diff
changeset
|
9 local set = require "prosody.util.set"; |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
10 local it = require "prosody.util.iterators"; |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
11 |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
12 local host = module.host; |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
13 |
13213
50324f66ca2a
plugins: Use integer config API with interval specification where sensible
Kim Alvefur <zash@zash.se>
parents:
13187
diff
changeset
|
14 local archive_item_limit = module:get_option_integer("storage_archive_item_limit", 10000, 0); |
13228
616c578c644f
mod_storage_internal: Use integer option method for cache size
Kim Alvefur <zash@zash.se>
parents:
13213
diff
changeset
|
15 local archive_item_count_cache = cache.new(module:get_option_integer("storage_archive_item_limit_cache_size", 1000, 1)); |
9883
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
16 |
13187
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
17 local use_shift = module:get_option_boolean("storage_archive_experimental_fast_delete", false); |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
18 |
5122
b41c33dc7c36
mod_storage_*: Don't explicitly set driver name, to ease copying/renaming modules.
Waqas Hussain <waqas20@gmail.com>
parents:
5121
diff
changeset
|
19 local driver = {}; |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
20 |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5133
diff
changeset
|
21 function driver:open(store, typ) |
8018
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
22 local mt = self[typ or "keyval"] |
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
23 if not mt then |
6283
7cf6d3a2c855
mod_storage_{none,internal,sql}: Return error for unsupported (everything but keyval) store types
Kim Alvefur <zash@zash.se>
parents:
5153
diff
changeset
|
24 return nil, "unsupported-store"; |
7cf6d3a2c855
mod_storage_{none,internal,sql}: Return error for unsupported (everything but keyval) store types
Kim Alvefur <zash@zash.se>
parents:
5153
diff
changeset
|
25 end |
8018
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
26 return setmetatable({ store = store, type = typ }, mt); |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
27 end |
8018
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
28 |
8020
342ce07836de
mod_storage_internal: Ignore unused 'self' argument [luacheck]
Kim Alvefur <zash@zash.se>
parents:
8019
diff
changeset
|
29 function driver:stores(username) -- luacheck: ignore 212/self |
8019
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8018
diff
changeset
|
30 return datamanager.stores(username, host); |
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8018
diff
changeset
|
31 end |
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8018
diff
changeset
|
32 |
8020
342ce07836de
mod_storage_internal: Ignore unused 'self' argument [luacheck]
Kim Alvefur <zash@zash.se>
parents:
8019
diff
changeset
|
33 function driver:purge(user) -- luacheck: ignore 212/self |
8019
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8018
diff
changeset
|
34 return datamanager.purge(user, host); |
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8018
diff
changeset
|
35 end |
925098aad268
mod_storage_internal: Reorder methods
Kim Alvefur <zash@zash.se>
parents:
8018
diff
changeset
|
36 |
8018
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
37 local keyval = { }; |
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
38 driver.keyval = { __index = keyval }; |
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
39 |
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
40 function keyval:get(user) |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
41 return datamanager.load(user, host, self.store); |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
42 end |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
43 |
8018
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
44 function keyval:set(user, data) |
4085
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
45 return datamanager.store(user, host, self.store, data); |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
46 end |
7699cef04740
storagemanager, mod_storage_internal: Split out default driver to mod_storage_internal, and greatly simplify storagemanager's error handling and fallback code
Matthew Wild <mwild1@gmail.com>
parents:
diff
changeset
|
47 |
8018
9545d0a9401f
mod_storage_internal: Separate driver from keyval implementation
Kim Alvefur <zash@zash.se>
parents:
6283
diff
changeset
|
48 function keyval:users() |
5153
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5133
diff
changeset
|
49 return datamanager.users(host, self.store, self.type); |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5133
diff
changeset
|
50 end |
688aeac0012a
mod_storage_internal, datamanager: Add support for iterating over users with data in a store
Kim Alvefur <zash@zash.se>
parents:
5133
diff
changeset
|
51 |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
52 local archive = {}; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
53 driver.archive = { __index = archive }; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
54 |
9884
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9883
diff
changeset
|
55 archive.caps = { |
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9883
diff
changeset
|
56 total = true; |
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9883
diff
changeset
|
57 quota = archive_item_limit; |
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9883
diff
changeset
|
58 truncate = true; |
11275
b8fada57faf0
mod_storage_internal: Add support for full ID range query
Kim Alvefur <zash@zash.se>
parents:
10926
diff
changeset
|
59 full_id_range = true; |
11278
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11275
diff
changeset
|
60 ids = true; |
9884
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9883
diff
changeset
|
61 }; |
9751c17f5281
mod_storage_internal,_sql: Expose archive capabilities feature set
Kim Alvefur <zash@zash.se>
parents:
9883
diff
changeset
|
62 |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
63 function archive:append(username, key, value, when, with) |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
64 when = when or now(); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
65 if not st.is_stanza(value) then |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
66 return nil, "unsupported-datatype"; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
67 end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
68 value = st.preserialize(st.clone(value)); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
69 value.when = when; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
70 value.with = with; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
71 value.attr.stamp = datetime.datetime(when); |
8309
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
72 |
9885
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
73 local cache_key = jid_join(username, host, self.store); |
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
74 local item_count = archive_item_count_cache:get(cache_key); |
9883
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
75 |
8309
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
76 if key then |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
77 local items, err = datamanager.list_load(username, host, self.store); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
78 if not items and err then return items, err; end |
9883
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
79 |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
80 -- Check the quota |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
81 item_count = items and #items or 0; |
9885
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
82 archive_item_count_cache:set(cache_key, item_count); |
9883
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
83 if item_count >= archive_item_limit then |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
84 module:log("debug", "%s reached or over quota, not adding to store", username); |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
85 return nil, "quota-limit"; |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
86 end |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
87 |
8309
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
88 if items then |
9883
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
89 -- Filter out any item with the same key as the one being added |
8309
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
90 items = array(items); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
91 items:filter(function (item) |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
92 return item.key ~= key; |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
93 end); |
9883
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
94 |
8309
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
95 value.key = key; |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
96 items:push(value); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
97 local ok, err = datamanager.list_store(username, host, self.store, items); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
98 if not ok then return ok, err; end |
9885
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
99 archive_item_count_cache:set(cache_key, #items); |
8309
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
100 return key; |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
101 end |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
102 else |
9883
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
103 if not item_count then -- Item count not cached? |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
104 -- We need to load the list to get the number of items currently stored |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
105 local items, err = datamanager.list_load(username, host, self.store); |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
106 if not items and err then return items, err; end |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
107 item_count = items and #items or 0; |
9885
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
108 archive_item_count_cache:set(cache_key, item_count); |
9883
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
109 end |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
110 if item_count >= archive_item_limit then |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
111 module:log("debug", "%s reached or over quota, not adding to store", username); |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
112 return nil, "quota-limit"; |
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
113 end |
8309
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
114 key = id(); |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
115 end |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
116 |
9895
6bd65bff03b5
mod_storage_internal: Include store name when reporting quota status
Kim Alvefur <zash@zash.se>
parents:
9888
diff
changeset
|
117 module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store); |
9883
f76bd399267c
mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
Matthew Wild <mwild1@gmail.com>
parents:
9105
diff
changeset
|
118 |
8309
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
119 value.key = key; |
5281c479955a
mod_storage_internal: Add support for archive key deduplication (like mod_storage_sql)
Kim Alvefur <zash@zash.se>
parents:
8173
diff
changeset
|
120 |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
121 local ok, err = datamanager.list_append(username, host, self.store, value); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
122 if not ok then return ok, err; end |
9885
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
123 archive_item_count_cache:set(cache_key, item_count+1); |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
124 return key; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
125 end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
126 |
13136
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
127 local function binary_search(haystack, test, min, max) |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
128 if min == nil then |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
129 min = 1; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
130 end |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
131 if max == nil then |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
132 max = #haystack; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
133 end |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
134 |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
135 local floor = math.floor; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
136 while min < max do |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
137 local mid = floor((max + min) / 2); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
138 |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
139 local result = test(haystack[mid]); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
140 if result < 0 then |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
141 max = mid; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
142 elseif result > 0 then |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
143 min = mid + 1; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
144 else |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
145 return mid, haystack[mid]; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
146 end |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
147 end |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
148 |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
149 return min, nil; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
150 end |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
151 |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
152 function archive:find(username, query) |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
153 local list, err = datamanager.list_open(username, host, self.store); |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
154 if not list then |
8173
3ff99d49082f
mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents:
8143
diff
changeset
|
155 if err then |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
156 return list, err; |
9997
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9903
diff
changeset
|
157 elseif query then |
10024
4a0d990253a0
mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents:
10018
diff
changeset
|
158 if query.before or query.after then |
4a0d990253a0
mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents:
10018
diff
changeset
|
159 return nil, "item-not-found"; |
4a0d990253a0
mod_storage_internal: Return appropriate error even with empty archive
Kim Alvefur <zash@zash.se>
parents:
10018
diff
changeset
|
160 end |
9997
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9903
diff
changeset
|
161 if query.total then |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
162 return function() |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
163 end, 0; |
9997
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9903
diff
changeset
|
164 end |
8173
3ff99d49082f
mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents:
8143
diff
changeset
|
165 end |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
166 return function() |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
167 end; |
8173
3ff99d49082f
mod_storage_internal: Return a noop iterator if archive is empty (fixes #920)
Kim Alvefur <zash@zash.se>
parents:
8143
diff
changeset
|
168 end |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
169 |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
170 local i = 0; |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
171 local iter = function() |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
172 i = i + 1; |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
173 return list[i] |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
174 end |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
175 |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
176 if query then |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
177 if query.reverse then |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
178 i = #list + 1 |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
179 iter = function() |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
180 i = i - 1 |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
181 return list[i] |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
182 end |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
183 end |
8089
4ba8cb75d925
mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
184 if query.key then |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
185 iter = it.filter(function(item) |
8089
4ba8cb75d925
mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
186 return item.key == query.key; |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
187 end, iter); |
8089
4ba8cb75d925
mod_storage_internal: Support the 'key' archive query field
Kim Alvefur <zash@zash.se>
parents:
8023
diff
changeset
|
188 end |
11278
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11275
diff
changeset
|
189 if query.ids then |
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11275
diff
changeset
|
190 local ids = set.new(query.ids); |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
191 iter = it.filter(function(item) |
11278
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11275
diff
changeset
|
192 return ids:contains(item.key); |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
193 end, iter); |
11278
c3907f05bed4
mod_storage_internal: Support query for set of IDs
Kim Alvefur <zash@zash.se>
parents:
11275
diff
changeset
|
194 end |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
195 if query.with then |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
196 iter = it.filter(function(item) |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
197 return item.with == query.with; |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
198 end, iter); |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
199 end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
200 if query.start then |
13136
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
201 if not query.reverse then |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
202 local wi, exact = binary_search(list, function(item) |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
203 local when = item.when or datetime.parse(item.attr.stamp); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
204 return query.start - when; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
205 end); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
206 if exact then |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
207 i = wi - 1; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
208 elseif wi then |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
209 i = wi; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
210 end |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
211 else |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
212 iter = it.filter(function(item) |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
213 local when = item.when or datetime.parse(item.attr.stamp); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
214 return when >= query.start; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
215 end, iter); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
216 end |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
217 end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
218 if query["end"] then |
13136
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
219 if query.reverse then |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
220 local wi = binary_search(list, function(item) |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
221 local when = item.when or datetime.parse(item.attr.stamp); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
222 return query["end"] - when; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
223 end); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
224 if wi then |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
225 i = wi + 1; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
226 end |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
227 else |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
228 iter = it.filter(function(item) |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
229 local when = item.when or datetime.parse(item.attr.stamp); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
230 return when <= query["end"]; |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
231 end, iter); |
396db0e7084f
mod_storage_internal: Use a binary search for time based ranges
Kim Alvefur <zash@zash.se>
parents:
13135
diff
changeset
|
232 end |
9997
7c4631d7b6fb
mod_storage_internal,memory: Only return total count if requested
Kim Alvefur <zash@zash.se>
parents:
9903
diff
changeset
|
233 end |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
234 if query.after then |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
235 local found = false; |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
236 iter = it.filter(function(item) |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
237 local found_after = found; |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
238 if item.key == query.after then |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
239 found = true |
10018
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10001
diff
changeset
|
240 end |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
241 return found_after; |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
242 end, iter); |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
243 end |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
244 if query.before then |
10018
7408b9473729
mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325)
Kim Alvefur <zash@zash.se>
parents:
10001
diff
changeset
|
245 local found = false; |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
246 iter = it.filter(function(item) |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
247 if item.key == query.before then |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
248 found = true |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
249 end |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
250 return not found; |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
251 end, iter); |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
252 end |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
253 if query.limit then |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
254 iter = it.head(query.limit, iter); |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
255 end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
256 end |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
257 |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
258 return function() |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
259 local item = iter(); |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
260 if item == nil then |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
261 return |
10926
c55bd98a54f8
mod_storage_internal, mod_storage_memory: Add support for query.before
Matthew Wild <mwild1@gmail.com>
parents:
10844
diff
changeset
|
262 end |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
263 local key = item.key; |
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
264 local when = item.when or item.attr and datetime.parse(item.attr.stamp); |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
265 local with = item.with; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
266 item.key, item.when, item.with = nil, nil, nil; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
267 item.attr.stamp = nil; |
11765
1cac469b18d0
mod_storage_internal: Stop storing XEP-0091 timestamp
Kim Alvefur <zash@zash.se>
parents:
11278
diff
changeset
|
268 -- COMPAT Stored data may still contain legacy XEP-0091 timestamp |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
269 item.attr.stamp_legacy = nil; |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
270 item = st.deserialize(item); |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
271 return key, item, when, with; |
13135
3fd24e1945b0
mod_storage_internal: Lazy-load archive items while iterating
Kim Alvefur <zash@zash.se>
parents:
12977
diff
changeset
|
272 end |
8021
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
273 end |
83f18982bcfd
mod_storage_internal: Add basic archive store implementation
Kim Alvefur <zash@zash.se>
parents:
8020
diff
changeset
|
274 |
10837
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
275 function archive:get(username, wanted_key) |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
276 local iter, err = self:find(username, { key = wanted_key }) |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
277 if not iter then return iter, err; end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
278 for key, stanza, when, with in iter do |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
279 if key == wanted_key then |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
280 return stanza, when, with; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
281 end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
282 end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
283 return nil, "item-not-found"; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
284 end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
285 |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
286 function archive:set(username, key, new_value, new_when, new_with) |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
287 local items, err = datamanager.list_load(username, host, self.store); |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
288 if not items then |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
289 if err then |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
290 return items, err; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
291 else |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
292 return nil, "item-not-found"; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
293 end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
294 end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
295 |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
296 for i = 1, #items do |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
297 local old_item = items[i]; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
298 if old_item.key == key then |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
299 local item = st.preserialize(st.clone(new_value)); |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
300 |
10842
5a6ba2f38e2b
mod_storage_internal: Fix keeping old timestamp in archive map API
Kim Alvefur <zash@zash.se>
parents:
10837
diff
changeset
|
301 local when = new_when or old_item.when or datetime.parse(old_item.attr.stamp); |
10837
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
302 item.key = key; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
303 item.when = when; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
304 item.with = new_with or old_item.with; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
305 item.attr.stamp = datetime.datetime(when); |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
306 items[i] = item; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
307 return datamanager.list_store(username, host, self.store, items); |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
308 end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
309 end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
310 |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
311 return nil, "item-not-found"; |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
312 end |
f23363380599
mod_storage_internal: Implement key-value API
Kim Alvefur <zash@zash.se>
parents:
10223
diff
changeset
|
313 |
8023
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8022
diff
changeset
|
314 function archive:dates(username) |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8022
diff
changeset
|
315 local items, err = datamanager.list_load(username, host, self.store); |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8022
diff
changeset
|
316 if not items then return items, err; end |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8022
diff
changeset
|
317 return array(items):pluck("when"):map(datetime.date):unique(); |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8022
diff
changeset
|
318 end |
5a9d491cc714
mod_storage_internal: Add the dates method
Kim Alvefur <zash@zash.se>
parents:
8022
diff
changeset
|
319 |
9903
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9895
diff
changeset
|
320 function archive:summary(username, query) |
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9895
diff
changeset
|
321 local iter, err = self:find(username, query) |
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9895
diff
changeset
|
322 if not iter then return iter, err; end |
10220
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10024
diff
changeset
|
323 local counts = {}; |
10222
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10221
diff
changeset
|
324 local earliest = {}; |
10221
068692cb9e78
mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10220
diff
changeset
|
325 local latest = {}; |
10223
d185c4961ee0
mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents:
10222
diff
changeset
|
326 local body = {}; |
d185c4961ee0
mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents:
10222
diff
changeset
|
327 for _, stanza, when, with in iter do |
10220
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10024
diff
changeset
|
328 counts[with] = (counts[with] or 0) + 1; |
10222
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10221
diff
changeset
|
329 if earliest[with] == nil then |
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10221
diff
changeset
|
330 earliest[with] = when; |
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10221
diff
changeset
|
331 end |
10221
068692cb9e78
mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10220
diff
changeset
|
332 latest[with] = when; |
10223
d185c4961ee0
mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents:
10222
diff
changeset
|
333 body[with] = stanza:get_child_text("body") or body[with]; |
9903
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9895
diff
changeset
|
334 end |
10220
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10024
diff
changeset
|
335 return { |
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10024
diff
changeset
|
336 counts = counts; |
10222
51f145094648
mod_storage_*: Also include timestmap of first message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10221
diff
changeset
|
337 earliest = earliest; |
10221
068692cb9e78
mod_storage_*: Include timestamp of latest message in :summary API
Kim Alvefur <zash@zash.se>
parents:
10220
diff
changeset
|
338 latest = latest; |
10223
d185c4961ee0
mod_storage_internal: Include last text message
Kim Alvefur <zash@zash.se>
parents:
10222
diff
changeset
|
339 body = body; |
10220
1e2b444acb72
mod_storage_*: Tweak :summary API to allow future expansion with more fields
Kim Alvefur <zash@zash.se>
parents:
10024
diff
changeset
|
340 }; |
9903
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9895
diff
changeset
|
341 end |
2c5546cc5c70
mod_storage_internal: Implement a summary API returning message counts per contact
Kim Alvefur <zash@zash.se>
parents:
9895
diff
changeset
|
342 |
10001
55f5588d71c6
mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents:
9997
diff
changeset
|
343 function archive:users() |
55f5588d71c6
mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents:
9997
diff
changeset
|
344 return datamanager.users(host, self.store, "list"); |
55f5588d71c6
mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents:
9997
diff
changeset
|
345 end |
55f5588d71c6
mod_storage_internal: Add support for iterating over users in archive stores
Kim Alvefur <zash@zash.se>
parents:
9997
diff
changeset
|
346 |
13187
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
347 function archive:trim(username, to_when) |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
348 local list, err = datamanager.list_open(username, host, self.store); |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
349 if not list then return list,err;end |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
350 -- luacheck: ignore 211/exact |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
351 local i, exact = binary_search(list, function(item) |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
352 local when = item.when or datetime.parse(item.attr.stamp); |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
353 return to_when - when; |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
354 end); |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
355 -- TODO if exact then ... off by one? |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
356 if i == 1 then return 0; end |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
357 local ok, err = datamanager.list_shift(username, host, self.store, i); |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
358 if not ok then return ok, err; end |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
359 return i-1; |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
360 end |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
361 |
8022
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
362 function archive:delete(username, query) |
9885
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
363 local cache_key = jid_join(username, host, self.store); |
8022
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
364 if not query or next(query) == nil then |
9885
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
365 archive_item_count_cache:set(cache_key, nil); |
8022
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
366 return datamanager.list_store(username, host, self.store, nil); |
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
367 end |
13187
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
368 |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
369 if use_shift and next(query) == "end" and next(query, "end") == nil then |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
370 return self:trim(username, query["end"]); |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
371 end |
fe1229919070
mod_storage_internal: Implement efficient deletion of oldest archive items
Kim Alvefur <zash@zash.se>
parents:
13136
diff
changeset
|
372 |
8022
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
373 local items, err = datamanager.list_load(username, host, self.store); |
8143
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8142
diff
changeset
|
374 if not items then |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8142
diff
changeset
|
375 if err then |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8142
diff
changeset
|
376 return items, err; |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8142
diff
changeset
|
377 end |
9885
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
378 archive_item_count_cache:set(cache_key, 0); |
8143
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8142
diff
changeset
|
379 -- Store is empty |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8142
diff
changeset
|
380 return 0; |
c4c159953c72
mod_storage_internal: Handle case of empty item store when deleting (fixes #910)
Kim Alvefur <zash@zash.se>
parents:
8142
diff
changeset
|
381 end |
8022
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
382 items = array(items); |
8142
13e075549cb3
mod_storage_internal: Correctly calculate number of deleted items (fixes #912)
Kim Alvefur <zash@zash.se>
parents:
8089
diff
changeset
|
383 local count_before = #items; |
8310
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
384 if query then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
385 if query.key then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
386 items:filter(function (item) |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
387 return item.key ~= query.key; |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
388 end); |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
389 end |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
390 if query.with then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
391 items:filter(function (item) |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
392 return item.with ~= query.with; |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
393 end); |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
394 end |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
395 if query.start then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
396 items:filter(function (item) |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
397 return item.when < query.start; |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
398 end); |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
399 end |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
400 if query["end"] then |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
401 items:filter(function (item) |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
402 return item.when > query["end"]; |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
403 end); |
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
404 end |
9105
e735c9865f42
mod_storage_internal: Only apply truncate if there are more items than requested
Kim Alvefur <zash@zash.se>
parents:
8402
diff
changeset
|
405 if query.truncate and #items > query.truncate then |
8393
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
406 if query.reverse then |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
407 -- Before: { 1, 2, 3, 4, 5, } |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
408 -- After: { 1, 2, 3 } |
8394
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8393
diff
changeset
|
409 for i = #items, query.truncate + 1, -1 do |
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8393
diff
changeset
|
410 items[i] = nil; |
8393
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
411 end |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
412 else |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
413 -- Before: { 1, 2, 3, 4, 5, } |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
414 -- After: { 3, 4, 5 } |
8394
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8393
diff
changeset
|
415 local offset = #items - query.truncate; |
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8393
diff
changeset
|
416 for i = 1, #items do |
4892c22403d5
mod_storage_internal: Optimize truncation
Kim Alvefur <zash@zash.se>
parents:
8393
diff
changeset
|
417 items[i] = items[i+offset]; |
8393
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
418 end |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
419 end |
b6a7b83f8d87
mod_storage_internal: Allow truncating deletion at the beginning or end of an archive store
Kim Alvefur <zash@zash.se>
parents:
8310
diff
changeset
|
420 end |
8310
1759491b53db
mod_storage_internal: Add more extensive query support to archive:delete method
Kim Alvefur <zash@zash.se>
parents:
8309
diff
changeset
|
421 end |
8142
13e075549cb3
mod_storage_internal: Correctly calculate number of deleted items (fixes #912)
Kim Alvefur <zash@zash.se>
parents:
8089
diff
changeset
|
422 local count = count_before - #items; |
8402
469afa02947b
mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents:
8394
diff
changeset
|
423 if count == 0 then |
469afa02947b
mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents:
8394
diff
changeset
|
424 return 0; -- No changes, skip write |
469afa02947b
mod_storage_internal: Skip write if no items matched a deletion query
Kim Alvefur <zash@zash.se>
parents:
8394
diff
changeset
|
425 end |
8022
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
426 local ok, err = datamanager.list_store(username, host, self.store, items); |
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
427 if not ok then return ok, err; end |
9885
64e16d1e91f6
mod_storage_internal,_sql: Key item count cache on both username and store
Kim Alvefur <zash@zash.se>
parents:
9884
diff
changeset
|
428 archive_item_count_cache:set(cache_key, #items); |
8022
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
429 return count; |
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
430 end |
05e201468f27
mod_storage_internal: Add support for removing archived items
Kim Alvefur <zash@zash.se>
parents:
8021
diff
changeset
|
431 |
5121
b5a5643f8572
core.storagemanager, mod_storage_*: "data-driver" -> "storage-provider", to allow using module:provides().
Waqas Hussain <waqas20@gmail.com>
parents:
5039
diff
changeset
|
432 module:provides("storage", driver); |