prosody
9c23e7c8a67a
plugins/mod_http_file_share.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Comparison

plugins/mod_http_file_share.lua @ 11781:9c23e7c8a67a

mod_http_file_share: Add optional global quota on total storage usage Before, maximum storage usage (assuming all users upload as much as they could) would depend on the quota, retention period and number of users. Since number of users can vary, this makes it hard to know how much storage will be needed. Adding a limit to the total overall storage use solves this, making it simple to set it to some number based on what storage is actually available. Summary job run less often than the prune job since it touches the entire archive; and started before the prune job since it's needed before the first upload.
author Kim Alvefur <zash@zash.se>
date Sun, 12 Sep 2021 01:38:33 +0200
parent 11611:a6d1131ac833
child 11784:f0971a9eba88
comparison
equal deleted inserted replaced
11780:98ae95235775 11781:9c23e7c8a67a
35 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB 35 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB
36 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {}); 36 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {});
37 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"}); 37 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
38 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400); 38 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
39 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day 39 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
40 local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil);
40 41
41 local access = module:get_option_set(module.name .. "_access", {}); 42 local access = module:get_option_set(module.name .. "_access", {});
42 43
43 if not external_base_url then 44 if not external_base_url then
44 module:depends("http"); 45 module:depends("http");
56 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large"; 57 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large";
57 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) }; 58 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) };
58 }; 59 };
59 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; }; 60 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
60 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; }; 61 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
62 unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
63 outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
61 }); 64 });
62 65
63 local upload_cache = cache.new(1024); 66 local upload_cache = cache.new(1024);
64 local quota_cache = cache.new(1024); 67 local quota_cache = cache.new(1024);
68
69 local total_storage_usage = nil;
65 70
66 local measure_upload_cache_size = module:measure("upload_cache", "amount"); 71 local measure_upload_cache_size = module:measure("upload_cache", "amount");
67 local measure_quota_cache_size = module:measure("quota_cache", "amount"); 72 local measure_quota_cache_size = module:measure("quota_cache", "amount");
68 73
69 module:hook_global("stats-update", function () 74 module:hook_global("stats-update", function ()
122 if not filesize or filesize < 0 or filesize % 1 ~= 0 then 127 if not filesize or filesize < 0 or filesize % 1 ~= 0 then
123 return false, upload_errors.new("filesizefmt"); 128 return false, upload_errors.new("filesizefmt");
124 end 129 end
125 if filesize > file_size_limit then 130 if filesize > file_size_limit then
126 return false, upload_errors.new("filesize"); 131 return false, upload_errors.new("filesize");
132 end
133
134 if total_storage_limit then
135 if not total_storage_usage then
136 return false, upload_errors.new("unknowntotal");
137 elseif total_storage_usage + filesize > total_storage_limit then
138 module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
139 return false, upload_errors.new("outofdisk");
140 end
127 end 141 end
128 142
129 local uploader_quota = get_daily_quota(uploader); 143 local uploader_quota = get_daily_quota(uploader);
130 if uploader_quota + filesize > daily_quota then 144 if uploader_quota + filesize > daily_quota then
131 return false, upload_errors.new("quota"); 145 return false, upload_errors.new("quota");
189 module:log("info", "Issuing upload slot to %s for %s", uploader, B(filesize)); 203 module:log("info", "Issuing upload slot to %s for %s", uploader, B(filesize));
190 local slot, storage_err = errors.coerce(uploads:append(nil, nil, request, os.time(), uploader)) 204 local slot, storage_err = errors.coerce(uploads:append(nil, nil, request, os.time(), uploader))
191 if not slot then 205 if not slot then
192 origin.send(st.error_reply(stanza, storage_err)); 206 origin.send(st.error_reply(stanza, storage_err));
193 return true; 207 return true;
208 end
209
210 if total_storage_usage then
211 total_storage_usage = total_storage_usage + filesize;
212 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
194 end 213 end
195 214
196 local cached_quota = quota_cache:get(uploader); 215 local cached_quota = quota_cache:get(uploader);
197 if cached_quota and cached_quota.time > os.time()-86400 then 216 if cached_quota and cached_quota.time > os.time()-86400 then
198 cached_quota.size = cached_quota.size + filesize; 217 cached_quota.size = cached_quota.size + filesize;
431 prune_done(); 450 prune_done();
432 return; 451 return;
433 end 452 end
434 453
435 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time)); 454 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
455 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
436 456
437 local obsolete_uploads = array(); 457 local obsolete_uploads = array();
438 local i = 0; 458 local i = 0;
439 for slot_id in iter do 459 local size_sum = 0;
460 for slot_id, slot_info in iter do
440 i = i + 1; 461 i = i + 1;
441 obsolete_uploads:push(slot_id); 462 obsolete_uploads:push(slot_id);
442 upload_cache:set(slot_id, nil); 463 upload_cache:set(slot_id, nil);
464 size_sum = size_sum + tonumber(slot_info.attr.size);
443 end 465 end
444 466
445 sleep(0.1); 467 sleep(0.1);
446 local n = 0; 468 local n = 0;
447 local problem_deleting = false; 469 local problem_deleting = false;
461 -- obsolete_uploads now contains slot ids for which the files have been 483 -- obsolete_uploads now contains slot ids for which the files have been
462 -- deleted and that needs to be cleared from the database 484 -- deleted and that needs to be cleared from the database
463 485
464 local deletion_query = {["end"] = boundary_time}; 486 local deletion_query = {["end"] = boundary_time};
465 if not problem_deleting then 487 if not problem_deleting then
466 module:log("info", "All (%d) expired files successfully deleted", n); 488 module:log("info", "All (%d, %s) expired files successfully deleted", n, B(size_sum));
489 if total_storage_usage then
490 total_storage_usage = total_storage_usage - size_sum;
491 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
492 end
467 -- we can delete based on time 493 -- we can delete based on time
468 else 494 else
469 module:log("warn", "%d out of %d expired files could not be deleted", n-#obsolete_uploads, n); 495 module:log("warn", "%d out of %d expired files could not be deleted", n-#obsolete_uploads, n);
470 -- we'll need to delete only those entries where the files were 496 -- we'll need to delete only those entries where the files were
471 -- successfully deleted, and then try again with the failed ones. 497 -- successfully deleted, and then try again with the failed ones.
472 -- eventually the admin ought to notice and fix the permissions or 498 -- eventually the admin ought to notice and fix the permissions or
473 -- whatever the problem is. 499 -- whatever the problem is.
500 -- total_storage_limit will be inaccurate until this has been resolved
474 deletion_query = {ids = obsolete_uploads}; 501 deletion_query = {ids = obsolete_uploads};
475 end 502 end
476 503
477 if #obsolete_uploads == 0 then 504 if #obsolete_uploads == 0 then
478 module:log("debug", "No metadata to remove"); 505 module:log("debug", "No metadata to remove");
487 end 514 end
488 515
489 prune_done(); 516 prune_done();
490 end); 517 end);
491 518
492 module:add_timer(1, function () 519 module:add_timer(5, function ()
493 reaper_task:run(os.time()-expiry); 520 reaper_task:run(os.time()-expiry);
494 return 60*60; 521 return 60*60;
522 end);
523 end
524
525 if total_storage_limit then
526 local async = require "util.async";
527
528 local summarizer_task = async.runner(function()
529 local summary_done = module:measure("summary", "times");
530 local iter = assert(uploads:find(nil));
531
532 local count, sum = 0, 0;
533 for _, file in iter do
534 sum = sum + tonumber(file.attr.size);
535 count = count + 1;
536 end
537
538 module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
539 total_storage_usage = sum;
540 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
541 summary_done();
542 end);
543
544 module:add_timer(1, function()
545 summarizer_task:run(true);
546 return 11 * 60 * 60;
495 end); 547 end);
496 end 548 end
497 549
498 -- Reachable from the console 550 -- Reachable from the console
499 function check_files(query) 551 function check_files(query)