Changeset

--- a/plugins/mod_http_file_share.lua	Sat Sep 11 22:24:34 2021 +0200
+++ b/plugins/mod_http_file_share.lua	Sun Sep 12 01:38:33 2021 +0200
@@ -37,6 +37,7 @@
 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
+local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil);
 
 local access = module:get_option_set(module.name .. "_access", {});
 
@@ -58,11 +59,15 @@
 	};
 	filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
 	quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
+	unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
+	outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
 });
 
 local upload_cache = cache.new(1024);
 local quota_cache = cache.new(1024);
 
+local total_storage_usage = nil;
+
 local measure_upload_cache_size = module:measure("upload_cache", "amount");
 local measure_quota_cache_size = module:measure("quota_cache", "amount");
 
@@ -126,6 +131,15 @@
 		return false, upload_errors.new("filesize");
 	end
 
+	if total_storage_limit then
+		if not total_storage_usage  then
+			return false, upload_errors.new("unknowntotal");
+		elseif total_storage_usage + filesize > total_storage_limit then
+			module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
+			return false, upload_errors.new("outofdisk");
+		end
+	end
+
 	local uploader_quota = get_daily_quota(uploader);
 	if uploader_quota + filesize > daily_quota then
 		return false, upload_errors.new("quota");
@@ -193,6 +207,11 @@
 		return true;
 	end
 
+	if total_storage_usage then
+		total_storage_usage = total_storage_usage + filesize;
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+	end
+
 	local cached_quota = quota_cache:get(uploader);
 	if cached_quota and cached_quota.time > os.time()-86400 then
 		cached_quota.size = cached_quota.size + filesize;
@@ -433,13 +452,16 @@
 		end
 
 		module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
 
 		local obsolete_uploads = array();
 		local i = 0;
-		for slot_id in iter do
+		local size_sum = 0;
+		for slot_id, slot_info in iter do
 			i = i + 1;
 			obsolete_uploads:push(slot_id);
 			upload_cache:set(slot_id, nil);
+			size_sum = size_sum + tonumber(slot_info.attr.size);
 		end
 
 		sleep(0.1);
@@ -463,7 +485,11 @@
 
 		local deletion_query = {["end"] = boundary_time};
 		if not problem_deleting then
-			module:log("info", "All (%d) expired files successfully deleted", n);
+			module:log("info", "All (%d, %s) expired files successfully deleted", n, B(size_sum));
+			if total_storage_usage then
+				total_storage_usage = total_storage_usage - size_sum;
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+			end
 			-- we can delete based on time
 		else
 			module:log("warn", "%d out of %d expired files could not be deleted", n-#obsolete_uploads, n);
@@ -471,6 +497,7 @@
 			-- successfully deleted, and then try again with the failed ones.
 			-- eventually the admin ought to notice and fix the permissions or
 			-- whatever the problem is.
+			-- total_storage_limit will be inaccurate until this has been resolved
 			deletion_query = {ids = obsolete_uploads};
 		end
 
@@ -489,12 +516,37 @@
 		prune_done();
 	end);
 
-	module:add_timer(1, function ()
+	module:add_timer(5, function ()
 		reaper_task:run(os.time()-expiry);
 		return 60*60;
 	end);
 end
 
+if total_storage_limit then
+	local async = require "util.async";
+
+	local summarizer_task = async.runner(function()
+		local summary_done = module:measure("summary", "times");
+		local iter = assert(uploads:find(nil));
+
+		local count, sum = 0, 0;
+		for _, file in iter do
+			sum = sum + tonumber(file.attr.size);
+			count = count + 1;
+		end
+
+		module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
+		total_storage_usage = sum;
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+		summary_done();
+	end);
+
+	module:add_timer(1, function()
+		summarizer_task:run(true);
+		return 11 * 60 * 60;
+	end);
+end
+
 -- Reachable from the console
 function check_files(query)
 	local issues = {};