prosody
5e68635cdc2c
plugins/mod_http_file_share.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Comparison

plugins/mod_http_file_share.lua @ 12179:5e68635cdc2c

mod_http_file_share: Always measure total disk usage for statistics! Metrics available or not depending on configuration is weird, even tho it might be expensive to calculate and it's only really needed when there is a global quota. Default quota is set to infinity, which is essentially what it was. Reports NaN if there is an error, which should count as over the infinite default quota.
author Kim Alvefur <zash@zash.se>
date Tue, 11 Jan 2022 04:15:29 +0100
parent 12008:c01532ae6a3b
child 12227:88958c0ecab3
comparison
equal deleted inserted replaced
12178:0aa99a6dfb3e 12179:5e68635cdc2c
18 local dt = require "util.datetime"; 18 local dt = require "util.datetime";
19 local hi = require "util.human.units"; 19 local hi = require "util.human.units";
20 local cache = require "util.cache"; 20 local cache = require "util.cache";
21 local lfs = require "lfs"; 21 local lfs = require "lfs";
22 22
23 local unknown = math.abs(0/0);
24 local unlimited = math.huge;
25
23 local namespace = "urn:xmpp:http:upload:0"; 26 local namespace = "urn:xmpp:http:upload:0";
24 27
25 module:depends("disco"); 28 module:depends("disco");
26 29
27 module:add_identity("store", "file", module:get_option_string("name", "HTTP File Upload")); 30 module:add_identity("store", "file", module:get_option_string("name", "HTTP File Upload"));
36 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB 39 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB
37 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {}); 40 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {});
38 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"}); 41 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
39 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400); 42 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
40 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day 43 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
41 local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil); 44 local total_storage_limit = module:get_option_number(module.name.."_global_quota", unlimited);
42 45
43 local access = module:get_option_set(module.name .. "_access", {}); 46 local access = module:get_option_set(module.name .. "_access", {});
44 47
45 if not external_base_url then 48 if not external_base_url then
46 module:depends("http"); 49 module:depends("http");
58 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large"; 61 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large";
59 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) }; 62 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) };
60 }; 63 };
61 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; }; 64 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
62 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; }; 65 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
63 unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
64 outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" }; 66 outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
65 }); 67 });
66 68
67 local upload_cache = cache.new(1024); 69 local upload_cache = cache.new(1024);
68 local quota_cache = cache.new(1024); 70 local quota_cache = cache.new(1024);
69 71
70 local total_storage_usage = nil; 72 local total_storage_usage = unknown;
71 73
72 local measure_upload_cache_size = module:measure("upload_cache", "amount"); 74 local measure_upload_cache_size = module:measure("upload_cache", "amount");
73 local measure_quota_cache_size = module:measure("quota_cache", "amount"); 75 local measure_quota_cache_size = module:measure("quota_cache", "amount");
74 local measure_total_storage_usage = nil; 76 local measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" });
75 if total_storage_limit then 77
78 do
76 local total, err = persist_stats:get(nil, "total"); 79 local total, err = persist_stats:get(nil, "total");
77 if not err then total_storage_usage = tonumber(total) or 0; end 80 if not err then
78 measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" }); 81 total_storage_usage = tonumber(total) or 0;
82 end
79 end 83 end
80 84
81 module:hook_global("stats-update", function () 85 module:hook_global("stats-update", function ()
82 measure_upload_cache_size(upload_cache:count()); 86 measure_upload_cache_size(upload_cache:count());
83 measure_quota_cache_size(quota_cache:count()); 87 measure_quota_cache_size(quota_cache:count());
84 if total_storage_limit and total_storage_usage then 88 measure_total_storage_usage(total_storage_usage);
85 measure_total_storage_usage(total_storage_usage);
86 end
87 end); 89 end);
88 90
89 local buckets = {}; 91 local buckets = {};
90 for n = 10, 40, 2 do 92 for n = 10, 40, 2 do
91 local exp = math.floor(2 ^ n); 93 local exp = math.floor(2 ^ n);
93 if exp >= file_size_limit then break end 95 if exp >= file_size_limit then break end
94 end 96 end
95 local measure_uploads = module:measure("upload", "sizes", {buckets = buckets}); 97 local measure_uploads = module:measure("upload", "sizes", {buckets = buckets});
96 98
97 -- Convenience wrapper for logging file sizes 99 -- Convenience wrapper for logging file sizes
98 local function B(bytes) return hi.format(bytes, "B", "b"); end 100 local function B(bytes)
101 if bytes ~= bytes then
102 return "unknown"
103 elseif bytes == unlimited then
104 return "unlimited";
105 end
106 return hi.format(bytes, "B", "b");
107 end
99 108
100 local function get_filename(slot, create) 109 local function get_filename(slot, create)
101 return dm.getpath(slot, module.host, module.name, "bin", create) 110 return dm.getpath(slot, module.host, module.name, "bin", create)
102 end 111 end
103 112
139 end 148 end
140 if filesize > file_size_limit then 149 if filesize > file_size_limit then
141 return false, upload_errors.new("filesize"); 150 return false, upload_errors.new("filesize");
142 end 151 end
143 152
144 if total_storage_limit then 153 if total_storage_usage + filesize > total_storage_limit then
145 if not total_storage_usage then 154 module:log("warn", "Global storage quota reached, at %s / %s!", B(total_storage_usage), B(total_storage_limit));
146 return false, upload_errors.new("unknowntotal"); 155 return false, upload_errors.new("outofdisk");
147 elseif total_storage_usage + filesize > total_storage_limit then
148 module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
149 return false, upload_errors.new("outofdisk");
150 end
151 end 156 end
152 157
153 local uploader_quota = get_daily_quota(uploader); 158 local uploader_quota = get_daily_quota(uploader);
154 if uploader_quota + filesize > daily_quota then 159 if uploader_quota + filesize > daily_quota then
155 return false, upload_errors.new("quota"); 160 return false, upload_errors.new("quota");
215 if not slot then 220 if not slot then
216 origin.send(st.error_reply(stanza, storage_err)); 221 origin.send(st.error_reply(stanza, storage_err));
217 return true; 222 return true;
218 end 223 end
219 224
220 if total_storage_usage then 225 total_storage_usage = total_storage_usage + filesize;
221 total_storage_usage = total_storage_usage + filesize; 226 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
222 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
223 end
224 227
225 local cached_quota = quota_cache:get(uploader); 228 local cached_quota = quota_cache:get(uploader);
226 if cached_quota and cached_quota.time > os.time()-86400 then 229 if cached_quota and cached_quota.time > os.time()-86400 then
227 cached_quota.size = cached_quota.size + filesize; 230 cached_quota.size = cached_quota.size + filesize;
228 quota_cache:set(uploader, cached_quota); 231 quota_cache:set(uploader, cached_quota);
470 prune_done(); 473 prune_done();
471 return; 474 return;
472 end 475 end
473 476
474 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time)); 477 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
475 if total_storage_usage then 478 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
476 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
477 elseif total_storage_limit then
478 module:log("debug", "Global quota %s / %s", "not yet calculated", B(total_storage_limit));
479 end
480 479
481 local obsolete_uploads = array(); 480 local obsolete_uploads = array();
482 local num_expired = 0; 481 local num_expired = 0;
483 local size_sum = 0; 482 local size_sum = 0;
484 local problem_deleting = false; 483 local problem_deleting = false;
511 -- eventually the admin ought to notice and fix the permissions or 510 -- eventually the admin ought to notice and fix the permissions or
512 -- whatever the problem is. 511 -- whatever the problem is.
513 deletion_query = {ids = obsolete_uploads}; 512 deletion_query = {ids = obsolete_uploads};
514 end 513 end
515 514
516 if total_storage_usage then 515 total_storage_usage = total_storage_usage - size_sum;
517 total_storage_usage = total_storage_usage - size_sum; 516 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
518 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); 517 persist_stats:set(nil, "total", total_storage_usage);
519 persist_stats:set(nil, "total", total_storage_usage);
520 end
521 518
522 if #obsolete_uploads == 0 then 519 if #obsolete_uploads == 0 then
523 module:log("debug", "No metadata to remove"); 520 module:log("debug", "No metadata to remove");
524 else 521 else
525 local removed, err = uploads:delete(nil, deletion_query); 522 local removed, err = uploads:delete(nil, deletion_query);
533 530
534 prune_done(); 531 prune_done();
535 end); 532 end);
536 end 533 end
537 534
538 if total_storage_limit then 535 local summary_start = module:measure("summary", "times");
539 local summary_start = module:measure("summary", "times"); 536
540 537 module:weekly("Calculate total storage usage", function()
541 module:weekly("Global quota check", function() 538 local summary_done = summary_start();
542 local summary_done = summary_start(); 539 local iter = assert(uploads:find(nil));
543 local iter = assert(uploads:find(nil)); 540
544 541 local count, sum = 0, 0;
545 local count, sum = 0, 0; 542 for _, file in iter do
546 for _, file in iter do 543 sum = sum + tonumber(file.attr.size);
547 sum = sum + tonumber(file.attr.size); 544 count = count + 1;
548 count = count + 1; 545 end
549 end 546
550 547 module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
551 module:log("info", "Uploaded files total: %s in %d files", B(sum), count); 548 if persist_stats:set(nil, "total", sum) then
552 total_storage_usage = sum; 549 total_storage_usage = sum;
553 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); 550 else
554 persist_stats:set(nil, "total", sum); 551 total_storage_usage = unknown;
555 summary_done(); 552 end
556 end); 553 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
557 554 summary_done();
558 end 555 end);
559 556
560 -- Reachable from the console 557 -- Reachable from the console
561 function check_files(query) 558 function check_files(query)
562 local issues = {}; 559 local issues = {};
563 local iter = assert(uploads:find(nil, query)); 560 local iter = assert(uploads:find(nil, query));