Changeset

--- a/mod_conversejs/mod_conversejs.lua	Mon Apr 24 20:56:56 2017 +0200
+++ b/mod_conversejs/mod_conversejs.lua	Mon Apr 24 20:57:23 2017 +0200
@@ -26,6 +26,7 @@
 				bosh_service_url = module:http_url("bosh","/http-bind");
 				websocket_url = has_ws and module:http_url("websocket","xmpp-websocket"):gsub("^http", "ws") or nil;
 				authentication = module:get_option_string("authentication") == "anonymous" and "anonymous" or "login";
+				jid = module.host;
 			}));
 		end;
 	}

--- a/mod_http_muc_log/mod_http_muc_log.lua	Mon Apr 24 20:56:56 2017 +0200
+++ b/mod_http_muc_log/mod_http_muc_log.lua	Mon Apr 24 20:57:23 2017 +0200
@@ -2,9 +2,7 @@
 local datetime = require"util.datetime";
 local jid_split = require"util.jid".split;
 local nodeprep = require"util.encodings".stringprep.nodeprep;
-local uuid = require"util.uuid".generate;
 local it = require"util.iterators";
-local gettime = require"socket".gettime;
 local url = require"socket.url";
 local os_time, os_date = os.time, os.date;
 local render = require"util.interpolation".new("%b{}", require"util.stanza".xml_escape);
@@ -140,7 +138,10 @@
 					weeks[#weeks+1] = { days = days };
 					current_day = 1;
 				end
-				days[current_day], current_day = { wday = tmp.wday, day = i, href = days_t[i] and datetime.date(days_t[i]) }, current_day+1;
+				days[current_day] = {
+					wday = tmp.wday, day = i, href = days_t[i] and datetime.date(days_t[i])
+				};
+				current_day = current_day+1;
 			end
 		end
 		table.sort(year, sort_m);
@@ -277,56 +278,10 @@
 	});
 end
 
-local cache = setmetatable({}, {__mode = 'v'});
-
-local function with_cache(f)
-	return function (event, path)
-		local request, response = event.request, event.response;
-		local ckey = path or "";
-		local cached = cache[ckey];
-
-		if cached then
-			local etag = cached.etag;
-			local if_none_match = request.headers.if_none_match;
-			if etag == if_none_match then
-				module:log("debug", "Client cache hit");
-				return 304;
-			end
-			module:log("debug", "Server cache hit");
-			response.headers.etag = etag;
-			response.headers.content_type = "text/html; charset=utf-8";
-			return cached[1];
-		end
-
-		local start = gettime();
-		local rendered = f(event, path);
-		module:log("debug", "Rendering took %dms", math.floor( (gettime() - start) * 1000 + 0.5));
-
-		if type(rendered) == "string" then
-			local etag = uuid();
-			cached = { rendered, etag = etag, date = datetime.date() };
-			response.headers.etag = etag;
-			cache[ckey] = cached;
-		end
-
-		response.headers.content_type = "text/html; charset=utf-8";
-		return rendered;
-	end
-end
-
--- How is cache invalidation a hard problem? ;)
-module:hook("muc-broadcast-message", function (event)
-	local room = event.room;
-	local room_name = jid_split(room.jid);
-	local today = datetime.date();
-	cache[get_link(room_name)] = nil;
-	cache[get_link(room_name, today)] = nil;
-end);
-
 module:provides("http", {
 	route = {
 		["GET /"] = list_rooms;
-		["GET /*"] = with_cache(logs_page);
+		["GET /*"] = logs_page;
 	};
 });
 

--- a/mod_http_upload/README.markdown	Mon Apr 24 20:56:56 2017 +0200
+++ b/mod_http_upload/README.markdown	Mon Apr 24 20:57:23 2017 +0200
@@ -27,6 +27,8 @@
 Limits
 ------
 
+### Max size
+
 A maximum file size can be set by:
 
 ``` {.lua}
@@ -37,6 +39,30 @@
 
 This can not be set over the value of `http_max_content_size` (default 10M).
 
+### Max age
+
+Files can be set to be deleted after some time:
+
+``` lua
+http_upload_expire_after = 60 * 60 * 24 * 7 -- a week in seconds
+```
+
+### User quota
+
+A total maximum size of all uploaded files per user can be set by:
+
+``` lua
+http_upload_quota = 1234 -- bytes
+```
+
+### File types
+
+Accepted file types can be limited by MIME type:
+
+``` lua
+http_upload_allowed_file_types = { "image/*", "text/plain" }
+```
+
 Path
 ----
 

--- a/mod_http_upload/mod_http_upload.lua	Mon Apr 24 20:56:56 2017 +0200
+++ b/mod_http_upload/mod_http_upload.lua	Mon Apr 24 20:57:23 2017 +0200
@@ -1,6 +1,6 @@
 -- mod_http_upload
 --
--- Copyright (C) 2015 Kim Alvefur
+-- Copyright (C) 2015-2017 Kim Alvefur
 --
 -- This file is MIT/X11 licensed.
 --
@@ -13,6 +13,7 @@
 local url = require "socket.url";
 local dataform = require "util.dataforms".new;
 local datamanager = require "util.datamanager";
+local array = require "util.array";
 local t_concat = table.concat;
 local t_insert = table.insert;
 local s_upper = string.upper;
@@ -28,6 +29,9 @@
 
 -- config
 local file_size_limit = module:get_option_number(module.name .. "_file_size_limit", 1024 * 1024); -- 1 MB
+local quota = module:get_option_number(module.name .. "_quota");
+local max_age = module:get_option_number(module.name .. "_expire_after");
+local allowed_file_types = module:get_option_set(module.name .. "_allowed_file_types");
 
 --- sanity
 local parser_body_limit = module:context("*"):get_option_number("http_max_content_size", 10*1024*1024);
@@ -41,6 +45,9 @@
 module:depends("http");
 module:depends("disco");
 
+local http_files = module:depends("http_files");
+local mime_map = module:shared("/*/http_files/mime").types;
+
 -- namespaces
 local namespace = "urn:xmpp:http:upload:0";
 local legacy_namespace = "urn:xmpp:http:upload";
@@ -66,7 +73,45 @@
 local storage_path = module:get_option_string(module.name .. "_path", join_path(prosody.paths.data, module.name));
 lfs.mkdir(storage_path);
 
-local function handle_request(origin, stanza, xmlns, filename, filesize)
+local function expire(username, host)
+	if not max_age then return true; end
+	local uploads, err = datamanager.list_load(username, host, module.name);
+	if not uploads then return true; end
+	uploads = array(uploads);
+	local expiry = os.time() - max_age;
+	local upload_window = os.time() - 900;
+	uploads:filter(function (item)
+		local filename = item.filename;
+		if item.dir then
+			filename = join_path(storage_path, item.dir, item.filename);
+		end
+		if item.time < expiry then
+			local deleted, whynot = os.remove(filename);
+			if not deleted then
+				module:log("warn", "Could not delete expired upload %s: %s", filename, whynot or "delete failed");
+			end
+			return false;
+		elseif item.time < upload_window and not lfs.attributes(filename) then
+			return false; -- File was not uploaded or has been deleted since
+		end
+		return true;
+	end);
+	return datamanager.list_store(username, host, module.name, uploads);
+end
+
+local function check_quota(username, host, does_it_fit)
+	if not quota then return true; end
+	local uploads, err = datamanager.list_load(username, host, module.name);
+	if not uploads then return true; end
+	local sum = does_it_fit or 0;
+	for _, item in ipairs(uploads) do
+		sum = sum + item.size;
+	end
+	return sum < quota;
+end
+
+local function handle_request(origin, stanza, xmlns, filename, filesize, mimetype)
+	local username, host = origin.username, origin.host;
 	-- local clients only
 	if origin.type ~= "c2s" then
 		module:log("debug", "Request for upload slot from a %s", origin.type);
@@ -79,6 +124,7 @@
 		origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid filename"));
 		return true;
 	end
+	expire(username, host);
 	if not filesize then
 		module:log("debug", "Missing file size");
 		origin.send(st.error_reply(stanza, "modify", "bad-request", "Missing or invalid file size"));
@@ -89,7 +135,34 @@
 			:tag("file-too-large", {xmlns=xmlns})
 				:tag("max-file-size"):text(tostring(file_size_limit)));
 		return true;
+	elseif not check_quota(username, host, filesize) then
+		module:log("debug", "Upload of %dB by %s would exceed quota", filesize, origin.full_jid);
+		origin.send(st.error_reply(stanza, "wait", "resource-constraint", "Quota reached"));
+		return true;
 	end
+
+	if mime_map then
+		local file_ext = filename:match("%.([^.]+)$");
+		if not mimetype then
+			mimetype = "application/octet-stream";
+			if file_ext then
+				mimetype = mime_map[file_ext] or mimetype;
+			end
+		else
+			if (not file_ext and mimetype ~= "application/octet-stream") or (file_ext and mime_map[file_ext] ~= mimetype) then
+				origin.send(st.error_reply(stanza, "modify", "bad-request", "MIME type does not match file extension"));
+				return true;
+			end
+		end
+	end
+
+	if allowed_file_types then
+		if not (allowed_file_types:contains(mimetype) or allowed_file_types:contains(mimetype:gsub("/.*", "/*"))) then
+			origin.send(st.error_reply(stanza, "cancel", "not-allowed", "File type not allowed"));
+			return true;
+		end
+	end
+
 	local reply = st.reply(stanza);
 	reply:tag("slot", { xmlns = xmlns });
 
@@ -98,10 +171,21 @@
 	until lfs.mkdir(join_path(storage_path, random_dir))
 		or not lfs.attributes(join_path(storage_path, random_dir, filename))
 
-	datamanager.list_append(origin.username, origin.host, module.name, {
-		filename = join_path(storage_path, random_dir, filename), size = filesize, time = os.time() });
+	local ok = datamanager.list_append(username, host, module.name, {
+		filename = filename, dir = random_dir, size = filesize, time = os.time() });
+
+	if not ok then
+		origin.send(st.error_reply(stanza, "wait", "internal-server-failure"));
+		return true;
+	end
+
 	local slot = random_dir.."/"..filename;
 	pending_slots[slot] = origin.full_jid;
+
+	module:add_timer(900, function()
+		pending_slots[slot] = nil;
+	end);
+
 	local base_url = module:http_url();
 	local slot_url = url.parse(base_url);
 	slot_url.path = url.parse_path(slot_url.path or "/");
@@ -123,7 +207,8 @@
 	local request = stanza.tags[1];
 	local filename = request.attr.filename;
 	local filesize = tonumber(request.attr.size);
-	return handle_request(origin, stanza, namespace, filename, filesize);
+	local mimetype = request.attr["content-type"];
+	return handle_request(origin, stanza, namespace, filename, filesize, mimetype);
 end);
 
 module:hook("iq/host/"..legacy_namespace..":request", function (event)
@@ -131,7 +216,8 @@
 	local request = stanza.tags[1];
 	local filename = request:get_child_text("filename");
 	local filesize = tonumber(request:get_child_text("size"));
-	return handle_request(origin, stanza, legacy_namespace, filename, filesize);
+	local mimetype = request:get_child_text("content-type");
+	return handle_request(origin, stanza, legacy_namespace, filename, filesize, mimetype);
 end);
 
 -- http service
@@ -217,7 +303,7 @@
 	end
 end
 
-local serve_uploaded_files = module:depends("http_files").serve(storage_path);
+local serve_uploaded_files = http_files.serve(storage_path);
 
 local function serve_head(event, path)
 	event.response.send = send_response_sans_body;

--- a/mod_log_auth/mod_log_auth.lua	Mon Apr 24 20:56:56 2017 +0200
+++ b/mod_log_auth/mod_log_auth.lua	Mon Apr 24 20:57:23 2017 +0200
@@ -1,15 +1,19 @@
 local mode = module:get_option_string("log_auth_ips", "failure");
-assert(({ all = true, failure = true, success = true })[mode], "Unknown log mode: "..tostring(mode).." - valid modes are 'all', 'failure', 'success'");
+assert(({ all = true, failure = true, success = true })[mode],
+	"Unknown log mode: "..tostring(mode).." - valid modes are 'all', 'failure', 'success'");
 
 if mode == "failure" or mode == "all" then
 	module:hook("authentication-failure", function (event)
-		module:log("info", "Failed authentication attempt (%s) for user %s from IP: %s", event.condition or "unknown-condition", event.session.username or "?", event.session.ip or "?");
+		local session = event.session;
+		local username = session.username or session.sasl_handler and session.sasl_handler.username or "?";
+		session.log("info", "Failed authentication attempt (%s) for user %s from IP: %s",
+			event.condition or "unknown-condition", username, session.ip or "?");
 	end);
 end
 
 if mode == "success" or mode == "all" then
 	module:hook("authentication-success", function (event)
 		local session = event.session;
-		module:log("info", "Successful authentication as %s from IP: %s", session.username, session.ip or "?");
+		session.log("info", "Successful authentication as %s from IP: %s", session.username, session.ip or "?");
 	end);
 end

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_log_http/README.markdown	Mon Apr 24 20:57:23 2017 +0200
@@ -0,0 +1,22 @@
+---
+summary: HTTP request logging
+...
+
+Introduction
+============
+
+This module logs *outgoing* requests that go via the internal net.http API.
+
+Output format liable to change.
+
+Configuration
+=============
+
+One option is required, set `log_http_file` to the file path you would like to log to.
+
+Compatibility
+=============
+
+  ----- -------
+  0.10   Works (requires 375cf924fce1 or later)
+  ----- -------

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_log_http/mod_log_http.lua	Mon Apr 24 20:57:23 2017 +0200
@@ -0,0 +1,78 @@
+module:set_global();
+
+local http = require "net.http";
+local codes = require "net.http.codes";
+local json = require "util.json";
+
+local log = assert(io.open(assert(module:get_option_string("log_http_file"), "Please supply log_http_file in the config"), "a+"));
+
+local function append_request(id, req)
+	local headers = {};
+	for k, v in pairs(req.headers) do
+		table.insert(headers, { name = k, value = v });
+	end
+	local queryString = {};
+	if req.query then
+		for _, pair in ipairs(http.formdecode(req.query)) do
+			table.insert(queryString, pair);
+		end
+	end
+	log:write("<<<", json.encode({
+		id = id;
+		type = "request";
+		method = req.method;
+		url = req.url;
+		httpVersion = "HTTP/1.1";
+		cookies = {};
+		headers = headers;
+		queryString = queryString;
+		postData = req.body and {
+			mimeType = req.headers["Content-Type"];
+			text = req.body;
+		} or nil;
+		headersSize = -1;
+		bodySize = -1;
+	}), "\n");
+end
+
+local function append_response(id, resp)
+	local headers = {};
+	for k, v in pairs(resp.headers) do
+		table.insert(headers, { name = k, value = v });
+	end
+	log:write(">>>", json.encode({
+		id = id;
+		type = "response";
+		status = resp.code;
+		statusText = codes[resp.code];
+		httpVersion = resp.httpversion;
+		cookies = {};
+		headers = headers;
+		content = resp.body and {
+			size = #resp.body;
+			mimeType = resp.headers.content_type;
+			text = resp.body;
+		} or nil;
+		headersSize = -1;
+		bodySize = -1;
+	}), "\n");
+end
+
+module:hook_object_event(http.events, "request", function (event)
+	module:log("warn", "Request to %s!", event.url);
+	append_request(event.request.id, event.request);
+end);
+
+module:hook_object_event(http.events, "request-connection-error", function (event)
+	module:log("warn", "Failed to make request to %s!", event.url);
+end);
+
+module:hook_object_event(http.events, "response", function (event)
+	module:log("warn", "Received response %d from %s!", event.code, event.url);
+	for k,v in pairs(event.response) do print("=====", k, v) end
+	append_response(event.request.id, event.response);
+end);
+
+function module.unload()
+	log:close();
+end

--- a/mod_measure_storage/mod_measure_storage.lua	Mon Apr 24 20:56:56 2017 +0200
+++ b/mod_measure_storage/mod_measure_storage.lua	Mon Apr 24 20:57:23 2017 +0200
@@ -14,7 +14,7 @@
 	else
 		metric_name = store_name.."_"..store_type.."_"..method_name;
 	end
-	local measure_operation_started = module:measure(metric_name, metric_tags);
+	local measure_operation_started = module:measure(metric_name, "times", metric_tags);
 
 	return function (...)
 		module:log("debug", "Measuring storage operation %s (%s)", metric_name, metric_tags or "no tags");
@@ -43,7 +43,7 @@
 
 local function hook_event(module)
 	module:hook("store-opened", function(event)
-		event.store = wrap_store(module, event.store_name, event.store_type, event.store);
+		event.store = wrap_store(module, event.store_name, event.store_type or "keyval", event.store);
 	end);
 end
 

--- a/mod_server_contact_info/mod_server_contact_info.lua	Mon Apr 24 20:56:56 2017 +0200
+++ b/mod_server_contact_info/mod_server_contact_info.lua	Mon Apr 24 20:57:23 2017 +0200
@@ -26,7 +26,7 @@
 		module:log("error", "No contact_info or admins set in config");
 		return -- Nothing to attach, so we'll just skip it.
 	end
-	module:log("debug", "No contact_info in config, using admins as fallback");
+	module:log("info", "No contact_info in config, using admins as fallback");
 	contact_config = {
 		admin = array.collect( admins / function(admin) return "xmpp:" .. admin; end);
 	};

--- a/mod_storage_xmlarchive/mod_storage_xmlarchive.lua	Mon Apr 24 20:56:56 2017 +0200
+++ b/mod_storage_xmlarchive/mod_storage_xmlarchive.lua	Mon Apr 24 20:57:23 2017 +0200
@@ -91,7 +91,7 @@
 	local stream_session = { notopen = true };
 	local stream_callbacks = { handlestanza = cb, stream_ns = "jabber:client", default_ns = "jabber:client" };
 	local stream = new_stream(stream_session, stream_callbacks);
-	local dates = self:dates() or empty;
+	local dates = self:dates(username) or empty;
 	local function reset_stream()
 		stream:reset();
 		stream_session.notopen = true;
@@ -279,14 +279,21 @@
 end
 
 function provider:purge(username)
-	for store in dm.stores(username, module.host) do
-		local dates = dm.list_load(username, module.host, store) or empty;
-		if dates[1] and type(dates[1]) == "string" and dates[1]:match("^%d%d%d%d%-%d%d%-%d%d$") then
-			module:log("info", "Store %s looks like an archive store, emptying it...", store);
-			provider:open(store, "archive"):delete(username);
+	local encoded_username = dm.path_encode((username or "@") .. "@");
+	local basepath = prosody.paths.data .. "/" .. dm.path_encode(module.host);
+	for store in lfs.dir(basepath) do
+		store = basepath .. "/" .. dm.path_encode(store);
+		if lfs.attributes(store, "mode") == "directory" then
+			for file in lfs.dir(store) do
+				if file:sub(1, #encoded_username) == encoded_username then
+					if file:sub(-4) == ".xml" or file:sub(-5) == ".list" then
+						os.remove(store .. "/" .. file);
+					end
+				end
+			end
+			return true;
 		end
 	end
-	return true;
 end
 
 module:provides("storage", provider);