prosody
fe1229919070
plugins/mod_storage_internal.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Comparison

plugins/mod_storage_internal.lua @ 13187:fe1229919070

mod_storage_internal: Implement efficient deletion of oldest archive items Using the new shift function in datamanager, either the oldest items are removed or all the later items are moved into a new file that replaces the old. Hidden behind a feature flag for now.
author Kim Alvefur <zash@zash.se>
date Wed, 12 Jul 2023 15:03:24 +0200
parent 13136:396db0e7084f
child 13213:50324f66ca2a
comparison
equal deleted inserted replaced
13186:affaf6d08d26 13187:fe1229919070
11 11
12 local host = module.host; 12 local host = module.host;
13 13
14 local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000); 14 local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000);
15 local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000)); 15 local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
16
17 local use_shift = module:get_option_boolean("storage_archive_experimental_fast_delete", false);
16 18
17 local driver = {}; 19 local driver = {};
18 20
19 function driver:open(store, typ) 21 function driver:open(store, typ)
20 local mt = self[typ or "keyval"] 22 local mt = self[typ or "keyval"]
340 342
341 function archive:users() 343 function archive:users()
342 return datamanager.users(host, self.store, "list"); 344 return datamanager.users(host, self.store, "list");
343 end 345 end
344 346
347 function archive:trim(username, to_when)
348 local list, err = datamanager.list_open(username, host, self.store);
349 if not list then return list,err;end
350 -- luacheck: ignore 211/exact
351 local i, exact = binary_search(list, function(item)
352 local when = item.when or datetime.parse(item.attr.stamp);
353 return to_when - when;
354 end);
355 -- TODO if exact then ... off by one?
356 if i == 1 then return 0; end
357 local ok, err = datamanager.list_shift(username, host, self.store, i);
358 if not ok then return ok, err; end
359 return i-1;
360 end
361
345 function archive:delete(username, query) 362 function archive:delete(username, query)
346 local cache_key = jid_join(username, host, self.store); 363 local cache_key = jid_join(username, host, self.store);
347 if not query or next(query) == nil then 364 if not query or next(query) == nil then
348 archive_item_count_cache:set(cache_key, nil); 365 archive_item_count_cache:set(cache_key, nil);
349 return datamanager.list_store(username, host, self.store, nil); 366 return datamanager.list_store(username, host, self.store, nil);
350 end 367 end
368
369 if use_shift and next(query) == "end" and next(query, "end") == nil then
370 return self:trim(username, query["end"]);
371 end
372
351 local items, err = datamanager.list_load(username, host, self.store); 373 local items, err = datamanager.list_load(username, host, self.store);
352 if not items then 374 if not items then
353 if err then 375 if err then
354 return items, err; 376 return items, err;
355 end 377 end