Changeset

--- a/util-src/crypto.c	Thu Sep 29 12:57:05 2022 +0100
+++ b/util-src/crypto.c	Thu Sep 29 23:15:39 2022 +0100
@@ -33,6 +33,9 @@
 #define luaL_setfuncs(L, R, N) luaL_register(L, NULL, R)
 #endif
 
+/* The max size of an encoded 'R' or 'S' value. P-521 = 521 bits = 66 bytes */
+#define MAX_ECDSA_SIG_INT_BYTES 66
+
 #include "managed_pointer.h"
 
 #define PKEY_MT_TAG "util.crypto key"
@@ -433,10 +436,15 @@
 	ECDSA_SIG *sig;
 	size_t sig_der_len;
 	const unsigned char *sig_der = (unsigned char*)luaL_checklstring(L, 1, &sig_der_len);
+	const size_t sig_int_bytes = luaL_checkinteger(L, 2);
 	const BIGNUM *r, *s;
-	unsigned char rb[32];
-	unsigned char sb[32];
 	int rlen, slen;
+	unsigned char rb[MAX_ECDSA_SIG_INT_BYTES];
+	unsigned char sb[MAX_ECDSA_SIG_INT_BYTES];
+
+	if(sig_int_bytes > MAX_ECDSA_SIG_INT_BYTES) {
+		luaL_error(L, "requested signature size exceeds supported limit");
+	}
 
 	sig = d2i_ECDSA_SIG(NULL, &sig_der, sig_der_len);
 
@@ -447,17 +455,14 @@
 
 	ECDSA_SIG_get0(sig, &r, &s);
 
-	rlen = BN_num_bytes(r);
-	slen = BN_num_bytes(s);
+	rlen = BN_bn2binpad(r, rb, sig_int_bytes);
+	slen = BN_bn2binpad(s, sb, sig_int_bytes);
 
-	if (rlen > 32 || slen > 32) {
+	if (rlen == -1 || slen == -1) {
 		ECDSA_SIG_free(sig);
-		luaL_error(L, "unexpectedly large signature integers");
+		luaL_error(L, "encoded integers exceed requested size");
 	}
 
-	BN_bn2bin(r, rb);
-	BN_bn2bin(s, sb);
-
 	ECDSA_SIG_free(sig);
 
 	lua_pushlstring(L, (const char*)rb, rlen);
@@ -485,7 +490,9 @@
 
 	luaL_buffinit(L, &sigbuf);
 
-	unsigned char *buffer = (unsigned char*)luaL_prepbuffsize(&sigbuf, rlen+slen+32);
+	/* DER structure of an ECDSA signature has 7 bytes plus the integers themselves,
+	   which may gain an extra byte once encoded */
+	unsigned char *buffer = (unsigned char*)luaL_prepbuffsize(&sigbuf, (rlen+1)+(slen+1)+7);
 	int len = i2d_ECDSA_SIG(sig, &buffer);
 	luaL_addsize(&sigbuf, len);
 	luaL_pushresult(&sigbuf);

--- a/util/jwt.lua	Thu Sep 29 12:57:05 2022 +0100
+++ b/util/jwt.lua	Thu Sep 29 23:15:39 2022 +0100
@@ -135,21 +135,21 @@
 end
 
 -- ES***
-local function new_ecdsa_algorithm(name, c_sign, c_verify)
+local function new_ecdsa_algorithm(name, c_sign, c_verify, sig_bytes)
 	local function encode_ecdsa_sig(der_sig)
-		local r, s = crypto.parse_ecdsa_signature(der_sig);
+		local r, s = crypto.parse_ecdsa_signature(der_sig, sig_bytes);
 		return r..s;
 	end
 
 	local function decode_ecdsa_sig(jwk_sig)
-		return crypto.build_ecdsa_signature(jwk_sig:sub(1, 32), jwk_sig:sub(33, 64));
+		return crypto.build_ecdsa_signature(jwk_sig:sub(1, sig_bytes), jwk_sig:sub(sig_bytes+1, sig_bytes*2));
 	end
 	return new_crypto_algorithm(name, "id-ecPublicKey", c_sign, c_verify, encode_ecdsa_sig, decode_ecdsa_sig);
 end
 
 local algorithms = {
 	HS256 = new_hmac_algorithm("HS256"), HS384 = new_hmac_algorithm("HS384"), HS512 = new_hmac_algorithm("HS512");
-	ES256 = new_ecdsa_algorithm("ES256", crypto.ecdsa_sha256_sign, crypto.ecdsa_sha256_verify);
+	ES256 = new_ecdsa_algorithm("ES256", crypto.ecdsa_sha256_sign, crypto.ecdsa_sha256_verify, 32);
 	RS256 = new_rsa_algorithm("RS256"), RS384 = new_rsa_algorithm("RS384"), RS512 = new_rsa_algorithm("RS512");
 	PS256 = new_rsa_algorithm("PS256"), PS384 = new_rsa_algorithm("PS384"), PS512 = new_rsa_algorithm("PS512");
 };