# HG changeset patch
# User Kim Alvefur <zash@zash.se>
# Date 1631403513 -7200
# Node ID 9c23e7c8a67a7b2b3c5f80ebd01104561bffb256
# Parent  98ae952357750348a1d3f594662ac6042713f691
mod_http_file_share: Add optional global quota on total storage usage

Before, maximum storage usage (assuming all users upload as much as they
could) would depend on the quota, retention period and number of users.
Since number of users can vary, this makes it hard to know how much
storage will be needed.

Adding a limit to the total overall storage use solves this, making it
simple to set it to some number based on what storage is actually
available.

Summary job run less often than the prune job since it touches the
entire archive; and started before the prune job since it's needed
before the first upload.

diff -r 98ae95235775 -r 9c23e7c8a67a plugins/mod_http_file_share.lua
--- a/plugins/mod_http_file_share.lua	Sat Sep 11 22:24:34 2021 +0200
+++ b/plugins/mod_http_file_share.lua	Sun Sep 12 01:38:33 2021 +0200
@@ -37,6 +37,7 @@
 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
+local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil);
 
 local access = module:get_option_set(module.name .. "_access", {});
 
@@ -58,11 +59,15 @@
 	};
 	filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
 	quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
+	unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
+	outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
 });
 
 local upload_cache = cache.new(1024);
 local quota_cache = cache.new(1024);
 
+local total_storage_usage = nil;
+
 local measure_upload_cache_size = module:measure("upload_cache", "amount");
 local measure_quota_cache_size = module:measure("quota_cache", "amount");
 
@@ -126,6 +131,15 @@
 		return false, upload_errors.new("filesize");
 	end
 
+	if total_storage_limit then
+		if not total_storage_usage  then
+			return false, upload_errors.new("unknowntotal");
+		elseif total_storage_usage + filesize > total_storage_limit then
+			module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
+			return false, upload_errors.new("outofdisk");
+		end
+	end
+
 	local uploader_quota = get_daily_quota(uploader);
 	if uploader_quota + filesize > daily_quota then
 		return false, upload_errors.new("quota");
@@ -193,6 +207,11 @@
 		return true;
 	end
 
+	if total_storage_usage then
+		total_storage_usage = total_storage_usage + filesize;
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+	end
+
 	local cached_quota = quota_cache:get(uploader);
 	if cached_quota and cached_quota.time > os.time()-86400 then
 		cached_quota.size = cached_quota.size + filesize;
@@ -433,13 +452,16 @@
 		end
 
 		module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
 
 		local obsolete_uploads = array();
 		local i = 0;
-		for slot_id in iter do
+		local size_sum = 0;
+		for slot_id, slot_info in iter do
 			i = i + 1;
 			obsolete_uploads:push(slot_id);
 			upload_cache:set(slot_id, nil);
+			size_sum = size_sum + tonumber(slot_info.attr.size);
 		end
 
 		sleep(0.1);
@@ -463,7 +485,11 @@
 
 		local deletion_query = {["end"] = boundary_time};
 		if not problem_deleting then
-			module:log("info", "All (%d) expired files successfully deleted", n);
+			module:log("info", "All (%d, %s) expired files successfully deleted", n, B(size_sum));
+			if total_storage_usage then
+				total_storage_usage = total_storage_usage - size_sum;
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+			end
 			-- we can delete based on time
 		else
 			module:log("warn", "%d out of %d expired files could not be deleted", n-#obsolete_uploads, n);
@@ -471,6 +497,7 @@
 			-- successfully deleted, and then try again with the failed ones.
 			-- eventually the admin ought to notice and fix the permissions or
 			-- whatever the problem is.
+			-- total_storage_limit will be inaccurate until this has been resolved
 			deletion_query = {ids = obsolete_uploads};
 		end
 
@@ -489,12 +516,37 @@
 		prune_done();
 	end);
 
-	module:add_timer(1, function ()
+	module:add_timer(5, function ()
 		reaper_task:run(os.time()-expiry);
 		return 60*60;
 	end);
 end
 
+if total_storage_limit then
+	local async = require "util.async";
+
+	local summarizer_task = async.runner(function()
+		local summary_done = module:measure("summary", "times");
+		local iter = assert(uploads:find(nil));
+
+		local count, sum = 0, 0;
+		for _, file in iter do
+			sum = sum + tonumber(file.attr.size);
+			count = count + 1;
+		end
+
+		module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
+		total_storage_usage = sum;
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+		summary_done();
+	end);
+
+	module:add_timer(1, function()
+		summarizer_task:run(true);
+		return 11 * 60 * 60;
+	end);
+end
+
 -- Reachable from the console
 function check_files(query)
 	local issues = {};