prosody
bf92f37de137%3A7934e2c6b030

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Changeset

9772:7934e2c6b030

Merge 0.11->trunk
author Kim Alvefur <zash@zash.se>
date Mon, 07 Jan 2019 15:02:11 +0100
parents 9766:f40b9649e929 (diff) 9771:bf92f37de137 (current diff)
children 9775:b16780e7939f
files
diffstat 47 files changed, 783 insertions(+), 210 deletions(-) [+]
line wrap: on
line diff
--- a/.luacheckrc	Sun Jan 06 21:09:32 2019 +0100
+++ b/.luacheckrc	Mon Jan 07 15:02:11 2019 +0100
@@ -1,7 +1,8 @@
 cache = true
 codes = true
-ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", "143/table", "113/unpack" }
+ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", }
 
+std = "lua53c"
 max_line_length = 150
 
 read_globals = {
@@ -79,6 +80,7 @@
 		"module.remove_item",
 		"module.require",
 		"module.send",
+		"module.send_iq",
 		"module.set_global",
 		"module.shared",
 		"module.unhook",
@@ -131,7 +133,6 @@
 	"fallbacks/bit.lua";
 	"fallbacks/lxp.lua";
 
-	"net/adns.lua";
 	"net/cqueues.lua";
 	"net/dns.lua";
 	"net/server_select.lua";
--- a/configure	Sun Jan 06 21:09:32 2019 +0100
+++ b/configure	Mon Jan 07 15:02:11 2019 +0100
@@ -237,7 +237,7 @@
    --lua-version|--with-lua-version)
       [ -n "$value" ] || die "Missing value in flag $key."
       LUA_VERSION="$value"
-      [ "$LUA_VERSION" = "5.1" ] || [ "$LUA_VERSION" = "5.2" ] || [ "$LUA_VERSION" = "5.3" ] || die "Invalid Lua version in flag $key."
+      [ "$LUA_VERSION" = "5.1" ] || [ "$LUA_VERSION" = "5.2" ] || [ "$LUA_VERSION" = "5.3" ] || [ "$LUA_VERSION" = "5.4" ] || die "Invalid Lua version in flag $key."
       LUA_VERSION_SET=yes
       ;;
    --with-lua)
@@ -340,7 +340,7 @@
 fi
 
 detect_lua_version() {
-   detected_lua=$("$1" -e 'print(_VERSION:match(" (5%.[123])$"))' 2> /dev/null)
+   detected_lua=$("$1" -e 'print(_VERSION:match(" (5%.[1234])$"))' 2> /dev/null)
    if [ "$detected_lua" != "nil" ]
    then
       if [ "$LUA_VERSION_SET" != "yes" ]
@@ -403,8 +403,14 @@
    elif [ "$LUA_VERSION_SET" = "yes" ] && [ "$LUA_VERSION" = "5.3" ]
    then
       suffixes="5.3 53 -5.3 -53"
+   elif [ "$LUA_VERSION_SET" = "yes" ] && [ "$LUA_VERSION" = "5.4" ]
+   then
+      suffixes="5.4 54 -5.4 -54"
    else
-      suffixes="5.1 51 -5.1 -51 5.2 52 -5.2 -52 5.3 53 -5.3 -53"
+      suffixes="5.1 51 -5.1 -51"
+      suffixes="$suffixes 5.2 52 -5.2 -52"
+      suffixes="$suffixes 5.3 53 -5.3 -53"
+      suffixes="$suffixes 5.4 54 -5.4 -54"
    fi
    for suffix in "" $suffixes
    do
@@ -457,30 +463,46 @@
    LUA_LIBDIR="$LUA_DIR/lib"
 fi
 
-echo_n "Checking Lua includes... "
 lua_h="$LUA_INCDIR/lua.h"
+echo_n "Looking for lua.h at $lua_h..."
 if [ -f "$lua_h" ]
 then
-   echo "lua.h found in $lua_h"
+   echo found
 else
-   v_dir="$LUA_INCDIR/lua/$LUA_VERSION"
-   lua_h="$v_dir/lua.h"
-   if [ -f "$lua_h" ]
-   then
-      echo "lua.h found in $lua_h"
+  echo "not found"
+  for postfix in "$LUA_VERSION" "$LUA_SUFFIX"; do
+    if ! [ "$postfix" = "" ]; then
+      v_dir="$LUA_INCDIR/lua/$postfix";
+    else
+      v_dir="$LUA_INCDIR/lua";
+    fi
+    lua_h="$v_dir/lua.h"
+    echo_n "Looking for lua.h at $lua_h..."
+    if [ -f "$lua_h" ]
+    then
       LUA_INCDIR="$v_dir"
-   else
-      d_dir="$LUA_INCDIR/lua$LUA_VERSION"
+      echo found
+      break;
+    else
+      echo "not found"
+      d_dir="$LUA_INCDIR/lua$postfix"
       lua_h="$d_dir/lua.h"
+      echo_n "Looking for lua.h at $lua_h..."
       if [ -f "$lua_h" ]
       then
-         echo "lua.h found in $lua_h (Debian/Ubuntu)"
-         LUA_INCDIR="$d_dir"
+        echo found
+        LUA_INCDIR="$d_dir"
+        break;
       else
-         echo "lua.h not found (looked in $LUA_INCDIR, $v_dir, $d_dir)"
-         die "You may want to use the flag --with-lua or --with-lua-include. See --help."
+        echo "not found"
       fi
-   fi
+    fi
+  done
+  if [ ! -f "$lua_h" ]; then
+    echo "lua.h not found."
+    echo
+    die "You may want to use the flag --with-lua or --with-lua-include. See --help."
+  fi
 fi
 
 if [ "$lua_interp_found" = "yes" ]
--- a/core/moduleapi.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/core/moduleapi.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -14,13 +14,16 @@
 local timer = require "util.timer";
 local resolve_relative_path = require"util.paths".resolve_relative_path;
 local st = require "util.stanza";
+local cache = require "util.cache";
+local errutil = require "util.error";
+local promise = require "util.promise";
 
 local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat;
 local error, setmetatable, type = error, setmetatable, type;
 local ipairs, pairs, select = ipairs, pairs, select;
 local tonumber, tostring = tonumber, tostring;
 local require = require;
-local pack = table.pack or function(...) return {n=select("#",...), ...}; end -- table.pack is only in 5.2
+local pack = table.pack or require "util.table".pack; -- table.pack is only in 5.2
 local unpack = table.unpack or unpack; --luacheck: ignore 113 -- renamed in 5.2
 
 local prosody = prosody;
@@ -361,6 +364,84 @@
 	return core_post_stanza(origin or hosts[self.host], stanza);
 end
 
+function api:send_iq(stanza, origin, timeout)
+	local iq_cache = self._iq_cache;
+	if not iq_cache then
+		iq_cache = cache.new(256, function (_, iq)
+			iq.reject(errutil.new({
+				type = "wait", condition = "resource-constraint",
+				text = "evicted from iq tracking cache"
+			}));
+			self:unhook(iq.result_event, iq.result_handler);
+			self:unhook(iq.error_event, iq.error_handler);
+		end);
+		self._iq_cache = iq_cache;
+	end
+	return promise.new(function (resolve, reject)
+		local event_type;
+		if stanza.attr.from == self.host then
+			event_type = "host";
+		else -- assume bare since we can't hook full jids
+			event_type = "bare";
+		end
+		local result_event = "iq-result/"..event_type.."/"..stanza.attr.id;
+		local error_event = "iq-error/"..event_type.."/"..stanza.attr.id;
+		local cache_key = event_type.."/"..stanza.attr.id;
+
+		local function result_handler(event)
+			if event.stanza.attr.from == stanza.attr.to then
+				resolve(event);
+				return true;
+			end
+		end
+
+		local function error_handler(event)
+			if event.stanza.attr.from == stanza.attr.to then
+				reject(errutil.from_stanza(event.stanza), event);
+				return true;
+			end
+		end
+
+		if iq_cache:get(cache_key) then
+			reject(errutil.new({
+				type = "modify", condition = "conflict",
+				text = "iq stanza id attribute already used",
+			}));
+			return;
+		end
+
+		self:hook(result_event, result_handler);
+		self:hook(error_event, error_handler);
+
+		local timeout_handle = self:add_timer(timeout or 120, function ()
+			reject(errutil.new({
+				type = "wait", condition = "remote-server-timeout",
+				text = "IQ stanza timed out",
+			}));
+			self:unhook(result_event, result_handler);
+			self:unhook(error_event, error_handler);
+			iq_cache:set(cache_key, nil);
+		end);
+
+		local ok = iq_cache:set(cache_key, {
+			reject = reject, resolve = resolve,
+			timeout_handle = timeout_handle,
+			result_event = result_event, error_event = error_event,
+			result_handler = result_handler, error_handler = error_handler;
+		});
+
+		if not ok then
+			reject(errutil.new({
+				type = "wait", condition = "internal-server-error",
+				text = "Could not store IQ tracking data"
+			}));
+			return;
+		end
+
+		self:send(stanza, origin);
+	end);
+end
+
 function api:broadcast(jids, stanza, iter)
 	for jid in (iter or it.values)(jids) do
 		local new_stanza = st.clone(stanza);
--- a/core/rostermanager.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/core/rostermanager.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -12,6 +12,7 @@
 local log = require "util.logger".init("rostermanager");
 
 local new_id = require "util.id".short;
+local new_cache = require "util.cache".new;
 
 local pairs = pairs;
 local tostring = tostring;
@@ -111,6 +112,23 @@
 	else -- Attempt to load roster for non-loaded user
 		log("debug", "load_roster: loading for offline user: %s", jid);
 	end
+	local roster_cache = hosts[host] and hosts[host].roster_cache;
+	if not roster_cache then
+		if hosts[host] then
+			roster_cache = new_cache(1024);
+			hosts[host].roster_cache = roster_cache;
+		end
+	else
+		roster = roster_cache:get(jid);
+		if roster then
+			log("debug", "load_roster: cache hit");
+			roster_cache:set(jid, roster);
+			if user then user.roster = roster; end
+			return roster;
+		else
+			log("debug", "load_roster: cache miss, loading from storage");
+		end
+	end
 	local roster_store = storagemanager.open(host, "roster", "keyval");
 	local data, err = roster_store:get(username);
 	roster = data or {};
@@ -134,6 +152,10 @@
 	if not err then
 		hosts[host].events.fire_event("roster-load", { username = username, host = host, roster = roster });
 	end
+	if roster_cache and not user then
+		log("debug", "load_roster: caching loaded roster");
+		roster_cache:set(jid, roster);
+	end
 	return roster, err;
 end
 
@@ -263,15 +285,15 @@
 
 function is_contact_pending_in(username, host, jid)
 	local roster = load_roster(username, host);
-	return roster[false].pending[jid];
+	return roster[false].pending[jid] ~= nil;
 end
-local function set_contact_pending_in(username, host, jid)
+local function set_contact_pending_in(username, host, jid, stanza)
 	local roster = load_roster(username, host);
 	local item = roster[jid];
 	if item and (item.subscription == "from" or item.subscription == "both") then
 		return; -- false
 	end
-	roster[false].pending[jid] = true;
+	roster[false].pending[jid] = st.is_stanza(stanza) and st.preserialize(stanza) or true;
 	return save_roster(username, host, roster, jid);
 end
 function is_contact_pending_out(username, host, jid)
--- a/makefile	Sun Jan 06 21:09:32 2019 +0100
+++ b/makefile	Mon Jan 07 15:02:11 2019 +0100
@@ -19,6 +19,9 @@
 MKDIR=install -d
 MKDIR_PRIVATE=$(MKDIR) -m750
 
+LUACHECK=luacheck
+BUSTED=busted
+
 .PHONY: all test clean install
 
 all: prosody.install prosodyctl.install prosody.cfg.lua.install prosody.version
@@ -68,8 +71,13 @@
 	rm -f prosody.version
 	$(MAKE) clean -C util-src
 
+lint:
+	$(LUACHECK) -q $$(HGPLAIN= hg files -I '**.lua') prosody prosodyctl
+	@echo $$(sed -n '/^\tlocal exclude_files/,/^}/p;' .luacheckrc | sed '1d;$d' | wc -l) files ignored
+	shellcheck configure
+
 test:
-	busted --lua=$(RUNWITH)
+	$(BUSTED) --lua=$(RUNWITH)
 
 
 prosody.install: prosody
--- a/net/adns.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/net/adns.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -14,7 +14,7 @@
 local coroutine, tostring, pcall = coroutine, tostring, pcall;
 local setmetatable = setmetatable;
 
-local function dummy_send(sock, data, i, j) return (j-i)+1; end
+local function dummy_send(sock, data, i, j) return (j-i)+1; end -- luacheck: ignore 212
 
 local _ENV = nil;
 -- luacheck: std none
@@ -29,8 +29,7 @@
 	local peername = "<unknown>";
 	local listener = {};
 	local handler = {};
-	local err;
-	function listener.onincoming(conn, data)
+	function listener.onincoming(conn, data) -- luacheck: ignore 212/conn
 		if data then
 			resolver:feed(handler, data);
 		end
@@ -46,9 +45,12 @@
 			resolver:servfail(conn); -- Let the magic commence
 		end
 	end
-	handler, err = server.wrapclient(sock, "dns", 53, listener);
-	if not handler then
-		return nil, err;
+	do
+		local err;
+		handler, err = server.wrapclient(sock, "dns", 53, listener);
+		if not handler then
+			return nil, err;
+		end
 	end
 
 	handler.settimeout = function () end
@@ -89,7 +91,7 @@
 			end)(resolver:peek(qname, qtype, qclass));
 end
 
-function query_methods:cancel(call_handler, reason)
+function query_methods:cancel(call_handler, reason) -- luacheck: ignore 212/reason
 	log("warn", "Cancelling DNS lookup for %s", tostring(self[4]));
 	self[1].cancel(self[2], self[3], self[4], self[5], call_handler);
 end
--- a/net/connlisteners.lua	Sun Jan 06 21:09:32 2019 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,18 +0,0 @@
--- COMPAT w/pre-0.9
-local log = require "util.logger".init("net.connlisteners");
-local traceback = debug.traceback;
-
-local _ENV = nil;
--- luacheck: std none
-
-local function fail()
-	log("error", "Attempt to use legacy connlisteners API. For more info see https://prosody.im/doc/developers/network");
-	log("error", "Legacy connlisteners API usage, %s", traceback("", 2));
-end
-
-return {
-	register = fail;
-	get = fail;
-	start = fail;
-	-- epic fail
-};
--- a/net/resolvers/basic.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/net/resolvers/basic.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -1,5 +1,6 @@
 local adns = require "net.adns";
 local inet_pton = require "util.net".pton;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 local methods = {};
 local resolver_mt = { __index = methods };
--- a/net/resolvers/manual.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/net/resolvers/manual.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -1,5 +1,6 @@
 local methods = {};
 local resolver_mt = { __index = methods };
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 -- Find the next target to connect to, and
 -- pass it to cb()
--- a/net/resolvers/service.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/net/resolvers/service.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -1,5 +1,6 @@
 local adns = require "net.adns";
 local basic = require "net.resolvers.basic";
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 local methods = {};
 local resolver_mt = { __index = methods };
--- a/net/server_epoll.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/net/server_epoll.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -120,9 +120,13 @@
 		end
 		local new_timeout = f(now);
 		if new_timeout then
-			-- Schedule for 'delay' from the time actually scheduled,
-			-- not from now, in order to prevent timer drift.
-			timer[1] = t + new_timeout;
+			-- Schedule for 'delay' from the time actually scheduled, not from now,
+			-- in order to prevent timer drift, unless it already drifted way out of sync.
+			if (t + new_timeout) > ( now - new_timeout ) then
+				timer[1] = t + new_timeout;
+			else
+				timer[1] = now + new_timeout;
+			end
 			resort_timers = true;
 		else
 			t_remove(timers, i);
@@ -176,6 +180,7 @@
 	local ok, err = pcall(listener, self, ...);
 	if not ok then
 		log("error", "Error calling on%s: %s", what, err);
+		return;
 	end
 	return err;
 end
@@ -423,8 +428,10 @@
 	else
 		self.writebuffer = { data };
 	end
-	self:setwritetimeout();
-	self:set(nil, true);
+	if not self._write_lock then
+		self:setwritetimeout();
+		self:set(nil, true);
+	end
 	return #data;
 end
 interface.send = interface.write;
@@ -571,6 +578,8 @@
 	client:init();
 	if self.tls_direct then
 		client:starttls(self.tls_ctx);
+	else
+		client:onconnect();
 	end
 end
 
@@ -604,11 +613,23 @@
 	end);
 end
 
+function interface:pause_writes()
+	self._write_lock = true;
+	self:setwritetimeout(false);
+	self:set(nil, false);
+end
+
+function interface:resume_writes()
+	self._write_lock = nil;
+	if self.writebuffer[1] then
+		self:setwritetimeout();
+		self:set(nil, true);
+	end
+end
+
 -- Connected!
 function interface:onconnect()
-	if self.conn and not self.peername and self.conn.getpeername then
-		self.peername, self.peerport = self.conn:getpeername();
-	end
+	self:updatenames();
 	self.onconnect = noop;
 	self:on("connect");
 end
--- a/net/server_event.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/net/server_event.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -253,6 +253,7 @@
 
 --TODO: Deprecate
 function interface_mt:lock_read(switch)
+	log("warn", ":lock_read is deprecated, use :pasue() and :resume()");
 	if switch then
 		return self:pause();
 	else
@@ -272,6 +273,19 @@
 	end
 end
 
+function interface_mt:pause_writes()
+	return self:_lock(self.nointerface, self.noreading, true);
+end
+
+function interface_mt:resume_writes()
+	self:_lock(self.nointerface, self.noreading, false);
+	if self.writecallback and not self.eventwrite then
+		self.eventwrite = addevent( base, self.conn, EV_WRITE, self.writecallback, cfg.WRITE_TIMEOUT );  -- register callback
+		return true;
+	end
+end
+
+
 function interface_mt:counter(c)
 	if c then
 		self._connections = self._connections + c
--- a/net/server_select.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/net/server_select.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -424,9 +424,8 @@
 		bufferlen = bufferlen + #data
 		if bufferlen > maxsendlen then
 			_closelist[ handler ] = "send buffer exceeded"	 -- cannot close the client at the moment, have to wait to the end of the cycle
-			handler.write = idfalse -- don't write anymore
 			return false
-		elseif socket and not _sendlist[ socket ] then
+		elseif not nosend and socket and not _sendlist[ socket ] then
 			_sendlistlen = addsocket(_sendlist, socket, _sendlistlen)
 		end
 		bufferqueuelen = bufferqueuelen + 1
@@ -456,49 +455,57 @@
 		maxreadlen = readlen or maxreadlen
 		return bufferlen, maxreadlen, maxsendlen
 	end
-	--TODO: Deprecate
 	handler.lock_read = function (self, switch)
+		out_error( "server.lua, lock_read() is deprecated, use pause() and resume()" )
 		if switch == true then
-			local tmp = _readlistlen
-			_readlistlen = removesocket( _readlist, socket, _readlistlen )
-			_readtimes[ handler ] = nil
-			if _readlistlen ~= tmp then
-				noread = true
-			end
+			return self:pause()
 		elseif switch == false then
-			if noread then
-				noread = false
-				_readlistlen = addsocket(_readlist, socket, _readlistlen)
-				_readtimes[ handler ] = _currenttime
-			end
+			return self:resume()
 		end
 		return noread
 	end
 	handler.pause = function (self)
-		return self:lock_read(true);
+		local tmp = _readlistlen
+		_readlistlen = removesocket( _readlist, socket, _readlistlen )
+		_readtimes[ handler ] = nil
+		if _readlistlen ~= tmp then
+			noread = true
+		end
+		return noread;
 	end
 	handler.resume = function (self)
-		return self:lock_read(false);
+		if noread then
+			noread = false
+			_readlistlen = addsocket(_readlist, socket, _readlistlen)
+			_readtimes[ handler ] = _currenttime
+		end
+		return noread;
 	end
 	handler.lock = function( self, switch )
-		handler.lock_read (switch)
+		out_error( "server.lua, lock() is deprecated" )
+		handler.lock_read (self, switch)
 		if switch == true then
-			handler.write = idfalse
-			local tmp = _sendlistlen
-			_sendlistlen = removesocket( _sendlist, socket, _sendlistlen )
-			_writetimes[ handler ] = nil
-			if _sendlistlen ~= tmp then
-				nosend = true
-			end
+			handler.pause_writes (self)
 		elseif switch == false then
-			handler.write = write
-			if nosend then
-				nosend = false
-				write( "" )
-			end
+			handler.resume_writes (self)
 		end
 		return noread, nosend
 	end
+	handler.pause_writes = function (self)
+		local tmp = _sendlistlen
+		_sendlistlen = removesocket( _sendlist, socket, _sendlistlen )
+		_writetimes[ handler ] = nil
+		if _sendlistlen ~= tmp then
+			nosend = true
+		end
+	end
+	handler.resume_writes = function (self)
+		if nosend then
+			nosend = false
+			write( "" )
+		end
+	end
+
 	local _readbuffer = function( ) -- this function reads data
 		local buffer, err, part = receive( socket, pattern )	-- receive buffer with "pattern"
 		if not err or (err == "wantread" or err == "timeout") then -- received something
--- a/net/websocket/frames.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/net/websocket/frames.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -9,20 +9,21 @@
 local softreq = require "util.dependencies".softreq;
 local random_bytes = require "util.random".bytes;
 
-local bit = assert(softreq"bit" or softreq"bit32",
+local bit = assert(softreq"bit32" or softreq"bit",
 	"No bit module found. See https://prosody.im/doc/depends#bitop");
 local band = bit.band;
 local bor = bit.bor;
 local bxor = bit.bxor;
 local lshift = bit.lshift;
 local rshift = bit.rshift;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 
 local t_concat = table.concat;
 local s_byte = string.byte;
 local s_char= string.char;
 local s_sub = string.sub;
-local s_pack = string.pack; -- luacheck: ignore 143
-local s_unpack = string.unpack; -- luacheck: ignore 143
+local s_pack = string.pack;
+local s_unpack = string.unpack;
 
 if not s_pack and softreq"struct" then
 	s_pack = softreq"struct".pack;
--- a/plugins/mod_admin_telnet.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_admin_telnet.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -22,6 +22,7 @@
 
 local console_listener = { default_port = 5582; default_mode = "*a"; interface = "127.0.0.1" };
 
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local iterators = require "util.iterators";
 local keys, values = iterators.keys, iterators.values;
 local jid_bare, jid_split, jid_join = import("util.jid", "bare", "prepped_split", "join");
@@ -30,6 +31,8 @@
 local envload = require "util.envload".envload;
 local envloadfile = require "util.envload".envloadfile;
 local has_pposix, pposix = pcall(require, "util.pposix");
+local async = require "util.async";
+local serialize = require "util.serialization".new({ fatal = false, unquoted = true});
 
 local commands = module:shared("commands")
 local def_env = module:shared("env");
@@ -47,6 +50,21 @@
 
 console = {};
 
+local runner_callbacks = {};
+
+function runner_callbacks:ready()
+	self.data.conn:resume();
+end
+
+function runner_callbacks:waiting()
+	self.data.conn:pause();
+end
+
+function runner_callbacks:error(err)
+	module:log("error", "Traceback[telnet]: %s", err);
+end
+
+
 function console:new_session(conn)
 	local w = function(s) conn:write(s:gsub("\n", "\r\n")); end;
 	local session = { conn = conn;
@@ -62,6 +80,11 @@
 			};
 	session.env = setmetatable({}, default_env_mt);
 
+	session.thread = async.runner(function (line)
+		console:process_line(session, line);
+		session.send(string.char(0));
+	end, runner_callbacks, session);
+
 	-- Load up environment with helper objects
 	for name, t in pairs(def_env) do
 		if type(t) == "table" then
@@ -150,8 +173,7 @@
 
 	for line in data:gmatch("[^\n]*[\n\004]") do
 		if session.closed then return end
-		console:process_line(session, line);
-		session.send(string.char(0));
+		session.thread:run(line);
 	end
 	session.partial_data = data:match("[^\n]+$");
 end
@@ -474,9 +496,12 @@
 	return true, "Config loaded";
 end
 
-function def_env.config:get(host, section, key)
+function def_env.config:get(host, key)
+	if key == nil then
+		host, key = "*", host;
+	end
 	local config_get = require "core.configmanager".get
-	return true, tostring(config_get(host, section, key));
+	return true, serialize(config_get(host, key));
 end
 
 function def_env.config:reload()
@@ -520,6 +545,9 @@
 	if session.remote then
 		line[#line+1] = "(remote)";
 	end
+	if session.is_bidi then
+		line[#line+1] = "(bidi)";
+	end
 	return table.concat(line, " ");
 end
 
@@ -1062,13 +1090,33 @@
 def_env.xmpp = {};
 
 local st = require "util.stanza";
-function def_env.xmpp:ping(localhost, remotehost)
-	if prosody.hosts[localhost] then
-		module:send(st.iq{ from=localhost, to=remotehost, type="get", id="ping" }
-				:tag("ping", {xmlns="urn:xmpp:ping"}), prosody.hosts[localhost]);
-		return true, "Sent ping";
+local new_id = require "util.id".medium;
+function def_env.xmpp:ping(localhost, remotehost, timeout)
+	localhost = select(2, jid_split(localhost));
+	remotehost = select(2, jid_split(remotehost));
+	if not localhost then
+		return nil, "Invalid sender hostname";
+	elseif not prosody.hosts[localhost] then
+		return nil, "No such local host";
+	end
+	if not remotehost then
+		return nil, "Invalid destination hostname";
+	elseif prosody.hosts[remotehost] then
+		return nil, "Both hosts are local";
+	end
+	local iq = st.iq{ from=localhost, to=remotehost, type="get", id=new_id()}
+			:tag("ping", {xmlns="urn:xmpp:ping"});
+	local ret, err;
+	local wait, done = async.waiter();
+	module:context(localhost):send_iq(iq, nil, timeout)
+		:next(function (ret_) ret = ret_; end,
+			function (err_) err = err_; end)
+		:finally(done);
+	wait();
+	if ret then
+		return true, "pong from " .. ret.stanza.attr.from;
 	else
-		return nil, "No such host";
+		return false, tostring(err);
 	end
 end
 
@@ -1495,7 +1543,7 @@
 	local stats, changed, extra = require "core.statsmanager".get_stats();
 	local available, displayed = 0, 0;
 	local displayed_stats = new_stats_context(self);
-	for name, value in pairs(stats) do
+	for name, value in iterators.sorted_pairs(stats) do
 		available = available + 1;
 		if not filter or name:match(filter) then
 			displayed = displayed + 1;
--- a/plugins/mod_c2s.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_c2s.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -106,7 +106,13 @@
 	if features.tags[1] or session.full_jid then
 		send(features);
 	else
-		(session.log or log)("warn", "No stream features to offer");
+		if session.secure then
+			-- Normally STARTTLS would be offered
+			(session.log or log)("warn", "No stream features to offer on secure session. Check authentication settings.");
+		else
+			-- Here SASL should be offered
+			(session.log or log)("warn", "No stream features to offer on insecure session. Check encryption and security settings.");
+		end
 		session:close{ condition = "undefined-condition", text = "No stream features to proceed with" };
 	end
 end
--- a/plugins/mod_http_errors.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_http_errors.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -26,21 +26,24 @@
 <meta charset="utf-8">
 <title>{title}</title>
 <style>
-body{
-	margin-top:14%;
-	text-align:center;
-	background-color:#F8F8F8;
-	font-family:sans-serif;
+body {
+	margin-top : 14%;
+	text-align : center;
+	background-color : #F8F8F8;
+	font-family : sans-serif
 }
-h1{
-	font-size:xx-large;
+
+h1 {
+	font-size : xx-large
 }
-p{
-	font-size:x-large;
+
+p {
+	font-size : x-large
 }
+
 p+p {
-	font-size:large;
-	font-family:courier;
+	font-size : large;
+	font-family : courier
 }
 </style>
 </head>
--- a/plugins/mod_mam/mod_mam.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_mam/mod_mam.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -33,7 +33,7 @@
 local tostring = tostring;
 local time_now = os.time;
 local m_min = math.min;
-local timestamp, timestamp_parse = require "util.datetime".datetime, require "util.datetime".parse;
+local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date");
 local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50);
 local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://jabber.org/protocol/chatstates" });
 
@@ -46,13 +46,8 @@
 end
 local use_total = module:get_option_boolean("mam_include_total", true);
 
-local cleanup;
-
-local function schedule_cleanup(username)
-	if cleanup and not cleanup[username] then
-		table.insert(cleanup, username);
-		cleanup[username] = true;
-	end
+function schedule_cleanup()
+	-- replaced by non-noop later if cleanup is enabled
 end
 
 -- Handle prefs.
@@ -96,7 +91,6 @@
 	local qid = query.attr.queryid;
 
 	get_prefs(origin.username, true);
-	schedule_cleanup(origin.username);
 
 	-- Search query parameters
 	local qwith, qstart, qend;
@@ -212,6 +206,7 @@
 local function shall_store(user, who)
 	-- TODO Cache this?
 	if not um.user_exists(user, host) then
+		module:log("debug", "%s@%s does not exist", user, host)
 		return false;
 	end
 	local prefs = get_prefs(user);
@@ -329,6 +324,9 @@
 local cleanup_after = module:get_option_string("archive_expires_after", "1w");
 local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
 if cleanup_after ~= "never" then
+	local cleanup_storage = module:open_store("archive_cleanup");
+	local cleanup_map = module:open_store("archive_cleanup", "map");
+
 	local day = 86400;
 	local multipliers = { d = day, w = day * 7, m = 31 * day, y = 365.2425 * day };
 	local n, m = cleanup_after:lower():match("(%d+)%s*([dwmy]?)");
@@ -346,33 +344,50 @@
 		return false;
 	end
 
-	-- Set of known users to do message expiry for
-	-- Populated either below or when new messages are added
-	cleanup = {};
+	-- For each day, store a set of users that have new messages. To expire
+	-- messages, we collect the union of sets of users from dates that fall
+	-- outside the cleanup range.
+
+	function schedule_cleanup(username, date)
+		cleanup_map:set(date or datestamp(), username, true);
+	end
+	local cleanup_time = module:measure("cleanup", "times");
 
-	-- Iterating over users is not supported by all authentication modules
-	-- Catch and ignore error if not supported
-	pcall(function ()
-		-- If this works, then we schedule cleanup for all known users on startup
-		for user in um.users(module.host) do
-			schedule_cleanup(user);
+	cleanup_runner = require "util.async".runner(function ()
+		local cleanup_done = cleanup_time();
+		local users = {};
+		local cut_off = datestamp(os.time() - cleanup_after);
+		for date in cleanup_storage:users() do
+			if date <= cut_off then
+				module:log("debug", "Messages from %q should be expired", date);
+				local messages_this_day = cleanup_storage:get(date);
+				if messages_this_day then
+					for user in pairs(messages_this_day) do
+						users[user] = true;
+					end
+					if date < cut_off then
+						-- Messages from the same day as the cut-off might not have expired yet,
+						-- but all earlier will have, so clear storage for those days.
+						cleanup_storage:set(date, nil);
+					end
+				end
+			end
 		end
+		local sum, num_users = 0, 0;
+		for user in pairs(users) do
+			local ok, err = archive:delete(user, { ["end"] = os.time() - cleanup_after; })
+			if ok then
+				num_users = num_users + 1;
+				sum = sum + tonumber(ok) or 0;
+			end
+		end
+		module:log("info", "Deleted %d expired messages for %d users", sum, num_users);
+		cleanup_done();
 	end);
 
-	-- At odd intervals, delete old messages for one user
-	module:add_timer(math.random(10, 60), function()
-		local user = table.remove(cleanup, 1);
-		if user then
-			module:log("debug", "Removing old messages for user %q", user);
-			local ok, err = archive:delete(user, { ["end"] = os.time() - cleanup_after; })
-			if not ok then
-				module:log("warn", "Could not expire archives for user %s: %s", user, err);
-			elseif type(ok) == "number" then
-				module:log("debug", "Removed %d messages", ok);
-			end
-			cleanup[user] = nil;
-		end
-		return math.random(cleanup_interval, cleanup_interval * 2);
+	cleanup_task = module:add_timer(1, function ()
+		cleanup_runner:run(true);
+		return cleanup_interval;
 	end);
 else
 	module:log("debug", "Archive expiry disabled");
--- a/plugins/mod_pep.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_pep.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -8,6 +8,7 @@
 local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed;
 local cache = require "util.cache";
 local set = require "util.set";
+local new_id = require "util.id".medium;
 
 local xmlns_pubsub = "http://jabber.org/protocol/pubsub";
 local xmlns_pubsub_event = "http://jabber.org/protocol/pubsub#event";
@@ -136,9 +137,7 @@
 		if kind == "retract" then
 			kind = "items"; -- XEP-0060 signals retraction in an <items> container
 		end
-		local message = st.message({ from = user_bare, type = "headline" })
-			:tag("event", { xmlns = xmlns_pubsub_event })
-				:tag(kind, { node = node });
+
 		if item then
 			item = st.clone(item);
 			item.attr.xmlns = nil; -- Clear the pubsub namespace
@@ -147,8 +146,17 @@
 					item:maptags(function () return nil; end);
 				end
 			end
+		end
+
+		local id = new_id();
+		local message = st.message({ from = user_bare, type = "headline", id = id })
+			:tag("event", { xmlns = xmlns_pubsub_event })
+				:tag(kind, { node = node });
+
+		if item then
 			message:add_child(item);
 		end
+
 		for jid in pairs(jids) do
 			module:log("debug", "Sending notification to %s from %s: %s", jid, user_bare, tostring(item));
 			message.attr.to = jid;
@@ -250,9 +258,6 @@
 module:hook("iq/bare/"..xmlns_pubsub..":pubsub", handle_pubsub_iq);
 module:hook("iq/bare/"..xmlns_pubsub_owner..":pubsub", handle_pubsub_iq);
 
-module:add_identity("pubsub", "pep", module:get_option_string("name", "Prosody"));
-module:add_feature("http://jabber.org/protocol/pubsub#publish");
-
 local function get_caps_hash_from_presence(stanza, current)
 	local t = stanza.attr.type;
 	if not t then
--- a/plugins/mod_pep_simple.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_pep_simple.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -14,6 +14,7 @@
 local pairs = pairs;
 local next = next;
 local type = type;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local calculate_hash = require "util.caps".calculate_hash;
 local core_post_stanza = prosody.core_post_stanza;
 local bare_sessions = prosody.bare_sessions;
--- a/plugins/mod_presence.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_presence.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -80,8 +80,14 @@
 				res.presence.attr.to = nil;
 			end
 		end
-		for jid in pairs(roster[false].pending) do -- resend incoming subscription requests
-			origin.send(st.presence({type="subscribe", from=jid})); -- TODO add to attribute? Use original?
+		for jid, pending_request in pairs(roster[false].pending) do -- resend incoming subscription requests
+			if type(pending_request) == "table" then
+				local subscribe = st.deserialize(pending_request);
+				subscribe.attr.type, subscribe.attr.from = "subscribe", jid;
+				origin.send(subscribe);
+			else
+				origin.send(st.presence({type="subscribe", from=jid}));
+			end
 		end
 		local request = st.presence({type="subscribe", from=origin.username.."@"..origin.host});
 		for jid, item in pairs(roster) do -- resend outgoing subscription requests
@@ -225,7 +231,7 @@
 		else
 			core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="unavailable"}), true); -- acknowledging receipt
 			if not rostermanager.is_contact_pending_in(node, host, from_bare) then
-				if rostermanager.set_contact_pending_in(node, host, from_bare) then
+				if rostermanager.set_contact_pending_in(node, host, from_bare, stanza) then
 					sessionmanager.send_to_available_resources(node, host, stanza);
 				end -- TODO else return error, unable to save
 			end
--- a/plugins/mod_pubsub/mod_pubsub.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_pubsub/mod_pubsub.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -73,7 +73,7 @@
 	local msg_type = node_obj and node_obj.config.message_type or "headline";
 	local message = st.message({ from = module.host, type = msg_type, id = id })
 		:tag("event", { xmlns = xmlns_pubsub_event })
-			:tag(kind, { node = node })
+			:tag(kind, { node = node });
 
 	if item then
 		message:add_child(item);
@@ -99,11 +99,12 @@
 end
 
 local max_max_items = module:get_option_number("pubsub_max_items", 256);
-function check_node_config(node, actor, new_config) -- luacheck: ignore 212/actor 212/node
+function check_node_config(node, actor, new_config) -- luacheck: ignore 212/node 212/actor
 	if (new_config["max_items"] or 1) > max_max_items then
 		return false;
 	end
-	if new_config["access_model"] ~= "whitelist" and new_config["access_model"] ~= "open" then
+	if new_config["access_model"] ~= "whitelist"
+	and new_config["access_model"] ~= "open" then
 		return false;
 	end
 	return true;
--- a/plugins/mod_saslauth.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_saslauth.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -275,7 +275,8 @@
 		if mechanisms[1] then
 			features:add_child(mechanisms);
 		elseif not next(sasl_mechanisms) then
-			log("warn", "No available SASL mechanisms, verify that the configured authentication module is working");
+			local authmod = module:get_option_string("authentication", "internal_plain");
+			log("error", "No available SASL mechanisms, verify that the configured authentication module '%s' is loaded and configured correctly", authmod);
 		else
 			log("warn", "All available authentication mechanisms are either disabled or not suitable for an insecure connection");
 		end
--- a/plugins/mod_storage_sql.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_storage_sql.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -11,7 +11,7 @@
 local t_concat = table.concat;
 
 local noop = function() end
-local unpack = table.unpack or unpack;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local function iterator(result)
 	return function(result_)
 		local row = result_();
--- a/plugins/mod_tls.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/mod_tls.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -35,9 +35,10 @@
 
 local ssl_ctx_c2s, ssl_ctx_s2sout, ssl_ctx_s2sin;
 local ssl_cfg_c2s, ssl_cfg_s2sout, ssl_cfg_s2sin;
+local err_c2s, err_s2sin, err_s2sout;
 
 function module.load()
-	local NULL, err = {};
+	local NULL = {};
 	local modhost = module.host;
 	local parent = modhost:match("%.(.*)$");
 
@@ -52,14 +53,14 @@
 	local parent_s2s = rawgetopt(parent,  "s2s_ssl") or NULL;
 	local host_s2s   = rawgetopt(modhost, "s2s_ssl") or parent_s2s;
 
-	ssl_ctx_c2s, err, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
-	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err); end
+	ssl_ctx_c2s, err_c2s, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
+	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err_c2s); end
 
-	ssl_ctx_s2sout, err, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s); -- for outgoing server connections
-	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err); end
+	ssl_ctx_s2sout, err_s2sout, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s); -- for outgoing server connections
+	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err_s2sout); end
 
-	ssl_ctx_s2sin, err, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s); -- for incoming server connections
-	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err); end
+	ssl_ctx_s2sin, err_s2sin, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s); -- for incoming server connections
+	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err_s2sin); end
 end
 
 module:hook_global("config-reloaded", module.load);
@@ -74,12 +75,21 @@
 		return session.ssl_ctx;
 	end
 	if session.type == "c2s_unauthed" then
+		if not ssl_ctx_c2s and c2s_require_encryption then
+			session.log("error", "No TLS context available for c2s. Earlier error was: %s", err_c2s);
+		end
 		session.ssl_ctx = ssl_ctx_c2s;
 		session.ssl_cfg = ssl_cfg_c2s;
 	elseif session.type == "s2sin_unauthed" and allow_s2s_tls then
+		if not ssl_ctx_s2sin and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sin. Earlier error was: %s", err_s2sin);
+		end
 		session.ssl_ctx = ssl_ctx_s2sin;
 		session.ssl_cfg = ssl_cfg_s2sin;
 	elseif session.direction == "outgoing" and allow_s2s_tls then
+		if not ssl_ctx_s2sout and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sout. Earlier error was: %s", err_s2sout);
+		end
 		session.ssl_ctx = ssl_ctx_s2sout;
 		session.ssl_cfg = ssl_cfg_s2sout;
 	else
--- a/plugins/muc/mod_muc.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/muc/mod_muc.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -453,7 +453,7 @@
 
 		if room == nil then
 			-- Watch presence to create rooms
-			if stanza.attr.type == nil and stanza.name == "presence" then
+			if stanza.attr.type == nil and stanza.name == "presence" and stanza:get_child("x", "http://jabber.org/protocol/muc") then
 				room = muclib.new_room(room_jid);
 				return room:handle_first_presence(origin, stanza);
 			elseif stanza.attr.type ~= "error" then
--- a/plugins/muc/muc.lib.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/muc/muc.lib.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -23,6 +23,7 @@
 local st = require "util.stanza";
 local base64 = require "util.encodings".base64;
 local md5 = require "util.hashes".md5;
+local new_id = require "util.id".medium;
 
 local log = module._log;
 
@@ -39,7 +40,7 @@
 end
 
 function room_mt.save()
-	-- overriden by mod_muc.lua
+	-- overridden by mod_muc.lua
 end
 
 function room_mt:get_occupant_jid(real_jid)
@@ -279,7 +280,7 @@
 		self_p = st.clone(base_presence):add_child(self_x);
 	end
 
-	-- General populance
+	-- General populace
 	for occupant_nick, n_occupant in self:each_occupant() do
 		if occupant_nick ~= occupant.nick then
 			local pr;
@@ -428,13 +429,6 @@
 end, 1);
 
 function room_mt:handle_first_presence(origin, stanza)
-	if not stanza:get_child("x", "http://jabber.org/protocol/muc") then
-		module:log("debug", "Room creation without <x>, possibly desynced");
-
-		origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
-		return true;
-	end
-
 	local real_jid = stanza.attr.from;
 	local dest_jid = stanza.attr.to;
 	local bare_jid = jid_bare(real_jid);
@@ -609,7 +603,7 @@
 				x:tag("status", {code = "303";}):up();
 				x:tag("status", {code = "110";}):up();
 				self:route_stanza(generated_unavail:add_child(x));
-				dest_nick = nil; -- set dest_nick to nil; so general populance doesn't see it for whole orig_occupant
+				dest_nick = nil; -- set dest_nick to nil; so general populace doesn't see it for whole orig_occupant
 			end
 		end
 
@@ -967,7 +961,7 @@
 	local _aff_rank = valid_affiliations[_aff or "none"];
 	local _rol = item.attr.role;
 	if _aff and _aff_rank and not _rol then
-		-- You need to be at least an admin, and be requesting info about your affifiliation or lower
+		-- You need to be at least an admin, and be requesting info about your affiliation or lower
 		-- e.g. an admin can't ask for a list of owners
 		local affiliation_rank = valid_affiliations[affiliation or "none"];
 		if (affiliation_rank >= valid_affiliations.admin and affiliation_rank >= _aff_rank)
@@ -1044,6 +1038,9 @@
 function room_mt:handle_groupchat_to_room(origin, stanza)
 	local from = stanza.attr.from;
 	local occupant = self:get_occupant_by_real_jid(from);
+	if not stanza.attr.id then
+		stanza.attr.id = new_id()
+	end
 	if module:fire_event("muc-occupant-groupchat", {
 		room = self; origin = origin; stanza = stanza; from = from; occupant = occupant;
 	}) then return true; end
@@ -1292,7 +1289,7 @@
 			-- Outcast can be by host.
 			is_host_only and affiliation == "outcast" and select(2, jid_split(occupant.bare_jid)) == host
 		) then
-			-- need to publcize in all cases; as affiliation in <item/> has changed.
+			-- need to publicize in all cases; as affiliation in <item/> has changed.
 			occupants_updated[occupant] = occupant.role;
 			if occupant.role ~= role and (
 				is_downgrade or
--- a/plugins/muc/subject.lib.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/plugins/muc/subject.lib.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -94,6 +94,12 @@
 	local stanza = event.stanza;
 	local subject = stanza:get_child("subject");
 	if subject then
+		if stanza:get_child("body") or stanza:get_child("thread") then
+			-- Note: A message with a <subject/> and a <body/> or a <subject/> and
+			-- a <thread/> is a legitimate message, but it SHALL NOT be interpreted
+			-- as a subject change.
+			return;
+		end
 		local room = event.room;
 		local occupant = event.occupant;
 		-- Role check for subject changes
--- a/spec/core_storagemanager_spec.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/spec/core_storagemanager_spec.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -1,4 +1,4 @@
-local unpack = table.unpack or unpack;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local server = require "net.server_select";
 package.loaded["net.server"] = server;
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/spec/scansion/keep_full_sub_req.scs	Mon Jan 07 15:02:11 2019 +0100
@@ -0,0 +1,58 @@
+# server MUST keep a record of the complete presence stanza comprising the subscription request (#689)
+
+[Client] Alice
+	jid: pars-a@localhost
+	password: password
+
+[Client] Bob
+	jid: pars-b@localhost
+	password: password
+
+[Client] Bob's phone
+	jid: pars-b@localhost/phone
+	password: password
+
+---------
+
+Alice connects
+
+Alice sends:
+	<presence to="${Bob's JID}" type="subscribe">
+		<preauth xmlns="urn:xmpp:pars:0" token="1tMFqYDdKhfe2pwp" />
+	</presence>
+
+Alice disconnects
+
+Bob connects
+
+Bob sends:
+	<presence/>
+
+Bob receives:
+	<presence from="${Bob's full JID}"/>
+	
+Bob receives:
+	<presence from="${Alice's JID}" type="subscribe">
+		<preauth xmlns="urn:xmpp:pars:0" token="1tMFqYDdKhfe2pwp" />
+	</presence>
+
+Bob disconnects
+
+# Works if they reconnect too
+
+Bob's phone connects
+
+Bob's phone sends:
+	<presence/>
+
+Bob's phone receives:
+	<presence from="${Bob's phone's full JID}"/>
+
+
+Bob's phone receives:
+	<presence from="${Alice's JID}" type="subscribe">
+		<preauth xmlns="urn:xmpp:pars:0" token="1tMFqYDdKhfe2pwp" />
+	</presence>
+
+Bob's phone disconnects
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/spec/scansion/muc_subject_issue_667.scs	Mon Jan 07 15:02:11 2019 +0100
@@ -0,0 +1,129 @@
+# #667 MUC message with subject and body SHALL NOT be interpreted as a subject change
+
+[Client] Romeo
+	password: password
+	jid: romeo@localhost
+
+-----
+
+Romeo connects
+
+# and creates a room
+Romeo sends:
+	<presence to="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+Romeo receives:
+	<presence from="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<status code="201"/>
+			<item affiliation="owner" role="moderator" jid="${Romeo's full JID}"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+# the default (empty) subject
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost">
+		<subject/>
+	</message>
+
+# this should be treated as a normal message
+Romeo sends:
+	<message to="issue667@conference.localhost" type="groupchat">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+# Resync
+Romeo sends:
+	<presence to="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+# Presences
+Romeo receives:
+	<presence from="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="owner" role="moderator" jid="${Romeo's full JID}"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+# the still empty subject
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost">
+		<subject/>
+	</message>
+
+# this is a subject change
+Romeo sends:
+	<message to="issue667@conference.localhost" type="groupchat">
+		<subject>Something to talk about</subject>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Something to talk about</subject>
+	</message>
+
+# a message without <subject>
+Romeo sends:
+	<message to="issue667@conference.localhost" type="groupchat">
+		<body>Lorem ipsum dolor sit amet</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<body>Lorem ipsum dolor sit amet</body>
+	</message>
+
+# Resync
+Romeo sends:
+	<presence to="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc"/>
+	</presence>
+
+# Presences
+Romeo receives:
+	<presence from="issue667@conference.localhost/Romeo">
+		<x xmlns="http://jabber.org/protocol/muc#user">
+			<item affiliation="owner" role="moderator" jid="${Romeo's full JID}"/>
+			<status code="110"/>
+		</x>
+	</presence>
+
+# History
+# These have delay tags but we ignore those for now
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Greetings</subject>
+		<body>Hello everyone</body>
+	</message>
+
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<body>Lorem ipsum dolor sit amet</body>
+	</message>
+
+# Finally, the topic
+Romeo receives:
+	<message type="groupchat" from="issue667@conference.localhost/Romeo">
+		<subject>Something to talk about</subject>
+	</message>
+
+Romeo disconnects
+
--- a/spec/scansion/prosody.cfg.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/spec/scansion/prosody.cfg.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -14,10 +14,11 @@
 
 	-- Not essential, but recommended
 		"carbons"; -- Keep multiple clients in sync
-		"pep"; -- Enables users to publish their mood, activity, playing music and more
+		"pep"; -- Enables users to publish their avatar, mood, activity, playing music and more
 		"private"; -- Private XML storage (for room bookmarks, etc.)
 		"blocklist"; -- Allow users to block communications with other users
-		"vcard"; -- Allow users to set vCards
+		"vcard4"; -- User profiles (stored in PEP)
+		"vcard_legacy"; -- Conversion between legacy vCard and PEP Avatar, vcard
 
 	-- Nice to have
 		"version"; -- Replies to server version requests
@@ -26,6 +27,11 @@
 		"ping"; -- Replies to XMPP pings with pongs
 		"register"; -- Allow users to register on this server using a client and change passwords
 		"mam"; -- Store messages in an archive and allow users to access it
+		--"csi_simple"; -- Simple Mobile optimizations
+
+	-- Admin interfaces
+		--"admin_adhoc"; -- Allows administration via an XMPP client that supports ad-hoc commands
+		--"admin_telnet"; -- Opens telnet console interface on localhost port 5582
 
 	-- HTTP modules
 		--"bosh"; -- Enable BOSH clients, aka "Jabber over HTTP"
--- a/spec/util_format_spec.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/spec/util_format_spec.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -5,10 +5,13 @@
 		it("should work", function()
 			assert.equal("hello", format("%s", "hello"));
 			assert.equal("<nil>", format("%s"));
+			assert.equal("<nil>", format("%d"));
+			assert.equal("<nil>", format("%q"));
 			assert.equal(" [<nil>]", format("", nil));
 			assert.equal("true", format("%s", true));
 			assert.equal("[true]", format("%d", true));
 			assert.equal("% [true]", format("%%", true));
+			assert.equal("{ }", format("%q", { }));
 		end);
 	end);
 end);
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/spec/util_interpolation_spec.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -0,0 +1,17 @@
+local template = [[
+{greet!}, {name?world}!
+]];
+local expect1 = [[
+Hello, WORLD!
+]];
+local expect2 = [[
+Hello, world!
+]];
+
+describe("util.interpolation", function ()
+	it("renders", function ()
+		local render = require "util.interpolation".new("%b{}", string.upper);
+		assert.equal(expect1, render(template, { greet = "Hello", name = "world" }));
+		assert.equal(expect2, render(template, { greet = "Hello" }));
+	end);
+end);
--- a/spec/util_stanza_spec.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/spec/util_stanza_spec.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -95,19 +95,30 @@
 
 	describe("#iq()", function()
 		it("should create an iq stanza", function()
-			local i = st.iq({ id = "foo" });
+			local i = st.iq({ type = "get", id = "foo" });
 			assert.are.equal("iq", i.name);
 			assert.are.equal("foo", i.attr.id);
+			assert.are.equal("get", i.attr.type);
 		end);
 
+		it("should reject stanzas with no attributes", function ()
+			assert.has.error_match(function ()
+				st.iq();
+			end, "attributes");
+		end);
+
+
 		it("should reject stanzas with no id", function ()
 			assert.has.error_match(function ()
-				st.iq();
+				st.iq({ type = "get" });
 			end, "id attribute");
+		end);
 
+		it("should reject stanzas with no type", function ()
 			assert.has.error_match(function ()
-				st.iq({ foo = "bar" });
-			end, "id attribute");
+				st.iq({ id = "foo" });
+			end, "type attribute");
+
 		end);
 	end);
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/spec/util_table_spec.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -0,0 +1,17 @@
+local u_table = require "util.table";
+describe("util.table", function ()
+	describe("create()", function ()
+		it("works", function ()
+			-- Can't test the allocated sizes of the table, so what you gonna do?
+			assert.is.table(u_table.create(1,1));
+		end);
+	end);
+
+	describe("pack()", function ()
+		it("works", function ()
+			assert.same({ "lorem", "ipsum", "dolor", "sit", "amet", n = 5 }, u_table.pack("lorem", "ipsum", "dolor", "sit", "amet"));
+		end);
+	end);
+end);
+
+
--- a/util-src/pposix.c	Sun Jan 06 21:09:32 2019 +0100
+++ b/util-src/pposix.c	Mon Jan 07 15:02:11 2019 +0100
@@ -25,14 +25,18 @@
 #define _DEFAULT_SOURCE
 #endif
 #endif
+
 #if defined(__APPLE__)
 #ifndef _DARWIN_C_SOURCE
 #define _DARWIN_C_SOURCE
 #endif
 #endif
+
+#if ! defined(__FreeBSD__)
 #ifndef _POSIX_C_SOURCE
 #define _POSIX_C_SOURCE 200809L
 #endif
+#endif
 
 #include <stdlib.h>
 #include <math.h>
--- a/util-src/time.c	Sun Jan 06 21:09:32 2019 +0100
+++ b/util-src/time.c	Mon Jan 07 15:02:11 2019 +0100
@@ -1,5 +1,5 @@
 #ifndef _POSIX_C_SOURCE
-#define _POSIX_C_SOURCE 199309L
+#define _POSIX_C_SOURCE 200809L
 #endif
 
 #include <time.h>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/util/error.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -0,0 +1,52 @@
+local error_mt = { __name = "error" };
+
+function error_mt:__tostring()
+	return ("error<%s:%s:%s>"):format(self.type, self.condition, self.text);
+end
+
+local function is_err(e)
+	return getmetatable(e) == error_mt;
+end
+
+local function new(e, context, registry)
+	local template = (registry and registry[e]) or e or {};
+	return setmetatable({
+		type = template.type or "cancel";
+		condition = template.condition or "undefined-condition";
+		text = template.text;
+
+		context = context or template.context or { _error_id = e };
+	}, error_mt);
+end
+
+local function coerce(ok, err, ...)
+	if ok or is_err(err) then
+		return ok, err, ...;
+	end
+
+	local new_err = setmetatable({
+		native = err;
+
+		type = "cancel";
+		condition = "undefined-condition";
+	}, error_mt);
+	return ok, new_err, ...;
+end
+
+local function from_stanza(stanza, context)
+	local error_type, condition, text = stanza:get_error();
+	return setmetatable({
+		type = error_type or "cancel";
+		condition = condition or "undefined-condition";
+		text = text;
+
+		context = context or { stanza = stanza };
+	}, error_mt);
+end
+
+return {
+	new = new;
+	coerce = coerce;
+	is_err = is_err;
+	from_stanza = from_stanza;
+}
--- a/util/format.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/util/format.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -3,12 +3,14 @@
 --
 
 local tostring = tostring;
-local select = select;
 local unpack = table.unpack or unpack; -- luacheck: ignore 113/unpack
+local pack = require "util.table".pack; -- TODO table.pack in 5.2+
 local type = type;
+local dump = require "util.serialization".new("debug");
 
 local function format(formatstring, ...)
-	local args, args_length = { ... }, select('#', ...);
+	local args = pack(...);
+	local args_length = args.n;
 
 	-- format specifier spec:
 	-- 1. Start: '%%'
@@ -28,13 +30,15 @@
 		if spec ~= "%%" then
 			i = i + 1;
 			local arg = args[i];
-			if arg == nil then -- special handling for nil
-				arg = "<nil>"
-				args[i] = "<nil>";
-			end
 
 			local option = spec:sub(-1);
-			if option == "q" or option == "s" then -- arg should be string
+			if arg == nil then
+				args[i] = "nil";
+				spec = "<%s>";
+			elseif option == "q" then
+				args[i] = dump(arg);
+				spec = "%s";
+			elseif option == "s" then
 				args[i] = tostring(arg);
 			elseif type(arg) ~= "number" then -- arg isn't number as expected?
 				args[i] = tostring(arg);
--- a/util/http.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/util/http.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -6,24 +6,25 @@
 --
 
 local format, char = string.format, string.char;
-local pairs, ipairs, tonumber = pairs, ipairs, tonumber;
+local pairs, ipairs = pairs, ipairs;
 local t_insert, t_concat = table.insert, table.concat;
 
+local url_codes = {};
+for i = 0, 255 do
+	local c = char(i);
+	local u = format("%%%02x", i);
+	url_codes[c] = u;
+	url_codes[u] = c;
+end
 local function urlencode(s)
-	return s and (s:gsub("[^a-zA-Z0-9.~_-]", function (c) return format("%%%02x", c:byte()); end));
+	return s and (s:gsub("[^a-zA-Z0-9.~_-]", url_codes));
 end
 local function urldecode(s)
-	return s and (s:gsub("%%(%x%x)", function (c) return char(tonumber(c,16)); end));
+	return s and (s:gsub("%%%x%x", url_codes));
 end
 
 local function _formencodepart(s)
-	return s and (s:gsub("%W", function (c)
-		if c ~= " " then
-			return format("%%%02x", c:byte());
-		else
-			return "+";
-		end
-	end));
+	return s and (urlencode(s):gsub("%%20", "+"));
 end
 
 local function formencode(form)
--- a/util/import.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/util/import.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -8,7 +8,7 @@
 
 
 
-local unpack = table.unpack or unpack; --luacheck: ignore 113 143
+local unpack = table.unpack or unpack; --luacheck: ignore 113
 local t_insert = table.insert;
 function _G.import(module, ...)
 	local m = package.loaded[module] or require(module);
--- a/util/iterators.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/util/iterators.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -11,9 +11,9 @@
 local it = {};
 
 local t_insert = table.insert;
-local select, next = select, next;
-local unpack = table.unpack or unpack; --luacheck: ignore 113 143
-local pack = table.pack or function (...) return { n = select("#", ...), ... }; end -- luacheck: ignore 143
+local next = next;
+local unpack = table.unpack or unpack; --luacheck: ignore 113
+local pack = table.pack or require "util.table".pack;
 local type = type;
 local table, setmetatable = table, setmetatable;
 
--- a/util/multitable.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/util/multitable.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -9,7 +9,7 @@
 local select = select;
 local t_insert = table.insert;
 local pairs, next, type = pairs, next, type;
-local unpack = table.unpack or unpack; --luacheck: ignore 113 143
+local unpack = table.unpack or unpack; --luacheck: ignore 113
 
 local _ENV = nil;
 -- luacheck: std none
--- a/util/promise.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/util/promise.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -49,6 +49,9 @@
 	for _, cb in ipairs(cbs) do
 		cb(value);
 	end
+	-- No need to keep references to callbacks
+	promise._pending_on_fulfilled = nil;
+	promise._pending_on_rejected = nil;
 	return true;
 end
 
--- a/util/serialization.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/util/serialization.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -20,7 +20,6 @@
 local envload = require"util.envload".envload;
 
 local pos_inf, neg_inf = math.huge, -math.huge;
--- luacheck: ignore 143/math
 local m_type = math.type or function (n)
 	return n % 1 == 0 and n <= 9007199254740992 and n >= -9007199254740992 and "integer" or "float";
 end;
--- a/util/stanza.lua	Sun Jan 06 21:09:32 2019 +0100
+++ b/util/stanza.lua	Mon Jan 07 15:02:11 2019 +0100
@@ -423,9 +423,15 @@
 	end
 end
 local function iq(attr)
-	if not (attr and attr.id) then
+	if not attr then
+		error("iq stanzas require id and type attributes");
+	end
+	if not attr.id then
 		error("iq stanzas require an id attribute");
 	end
+	if not attr.type then
+		error("iq stanzas require a type attribute");
+	end
 	return new_stanza("iq", attr);
 end