prosody
f76bd399267c
plugins/mod_storage_internal.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Comparison

plugins/mod_storage_internal.lua @ 9883:f76bd399267c

mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733)
author Matthew Wild <mwild1@gmail.com>
date Fri, 20 Oct 2017 12:53:53 +0200
parent 9105:e735c9865f42
child 9884:9751c17f5281
comparison
equal deleted inserted replaced
9882:18f025b3987d 9883:f76bd399267c
1 local cache = require "util.cache";
1 local datamanager = require "core.storagemanager".olddm; 2 local datamanager = require "core.storagemanager".olddm;
2 local array = require "util.array"; 3 local array = require "util.array";
3 local datetime = require "util.datetime"; 4 local datetime = require "util.datetime";
4 local st = require "util.stanza"; 5 local st = require "util.stanza";
5 local now = require "util.time".now; 6 local now = require "util.time".now;
6 local id = require "util.id".medium; 7 local id = require "util.id".medium;
7 8
8 local host = module.host; 9 local host = module.host;
9 10
11 local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000);
12 local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
13
10 local driver = {}; 14 local driver = {};
11 15
12 function driver:open(store, typ) 16 function driver:open(store, typ)
13 local mt = self[typ or "keyval"] 17 local mt = self[typ or "keyval"]
14 if not mt then 18 if not mt then
52 value.when = when; 56 value.when = when;
53 value.with = with; 57 value.with = with;
54 value.attr.stamp = datetime.datetime(when); 58 value.attr.stamp = datetime.datetime(when);
55 value.attr.stamp_legacy = datetime.legacy(when); 59 value.attr.stamp_legacy = datetime.legacy(when);
56 60
61 local item_count = archive_item_count_cache:get(username);
62
57 if key then 63 if key then
58 local items, err = datamanager.list_load(username, host, self.store); 64 local items, err = datamanager.list_load(username, host, self.store);
59 if not items and err then return items, err; end 65 if not items and err then return items, err; end
66
67 -- Check the quota
68 item_count = items and #items or 0;
69 archive_item_count_cache:set(username, item_count);
70 if item_count >= archive_item_limit then
71 module:log("debug", "%s reached or over quota, not adding to store", username);
72 return nil, "quota-limit";
73 end
74
60 if items then 75 if items then
76 -- Filter out any item with the same key as the one being added
61 items = array(items); 77 items = array(items);
62 items:filter(function (item) 78 items:filter(function (item)
63 return item.key ~= key; 79 return item.key ~= key;
64 end); 80 end);
81
65 value.key = key; 82 value.key = key;
66 items:push(value); 83 items:push(value);
67 local ok, err = datamanager.list_store(username, host, self.store, items); 84 local ok, err = datamanager.list_store(username, host, self.store, items);
68 if not ok then return ok, err; end 85 if not ok then return ok, err; end
86 archive_item_count_cache:set(username, #items);
69 return key; 87 return key;
70 end 88 end
71 else 89 else
90 if not item_count then -- Item count not cached?
91 -- We need to load the list to get the number of items currently stored
92 local items, err = datamanager.list_load(username, host, self.store);
93 if not items and err then return items, err; end
94 item_count = items and #items or 0;
95 archive_item_count_cache:set(username, item_count);
96 end
97 if item_count >= archive_item_limit then
98 module:log("debug", "%s reached or over quota, not adding to store", username);
99 return nil, "quota-limit";
100 end
72 key = id(); 101 key = id();
73 end 102 end
103
104 module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit);
74 105
75 value.key = key; 106 value.key = key;
76 107
77 local ok, err = datamanager.list_append(username, host, self.store, value); 108 local ok, err = datamanager.list_append(username, host, self.store, value);
78 if not ok then return ok, err; end 109 if not ok then return ok, err; end
110 archive_item_count_cache:set(username, item_count+1);
79 return key; 111 return key;
80 end 112 end
81 113
82 function archive:find(username, query) 114 function archive:find(username, query)
83 local items, err = datamanager.list_load(username, host, self.store); 115 local items, err = datamanager.list_load(username, host, self.store);
156 return array(items):pluck("when"):map(datetime.date):unique(); 188 return array(items):pluck("when"):map(datetime.date):unique();
157 end 189 end
158 190
159 function archive:delete(username, query) 191 function archive:delete(username, query)
160 if not query or next(query) == nil then 192 if not query or next(query) == nil then
193 archive_item_count_cache:set(username, nil);
161 return datamanager.list_store(username, host, self.store, nil); 194 return datamanager.list_store(username, host, self.store, nil);
162 end 195 end
163 local items, err = datamanager.list_load(username, host, self.store); 196 local items, err = datamanager.list_load(username, host, self.store);
164 if not items then 197 if not items then
165 if err then 198 if err then
166 return items, err; 199 return items, err;
167 end 200 end
201 archive_item_count_cache:set(username, 0);
168 -- Store is empty 202 -- Store is empty
169 return 0; 203 return 0;
170 end 204 end
171 items = array(items); 205 items = array(items);
172 local count_before = #items; 206 local count_before = #items;
212 if count == 0 then 246 if count == 0 then
213 return 0; -- No changes, skip write 247 return 0; -- No changes, skip write
214 end 248 end
215 local ok, err = datamanager.list_store(username, host, self.store, items); 249 local ok, err = datamanager.list_store(username, host, self.store, items);
216 if not ok then return ok, err; end 250 if not ok then return ok, err; end
251 archive_item_count_cache:set(username, #items);
217 return count; 252 return count;
218 end 253 end
219 254
220 module:provides("storage", driver); 255 module:provides("storage", driver);