prosody
7076ed654ac9
net/http/parser.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Comparison

net/http/parser.lua @ 11020:7076ed654ac9

net.http.parser: Switch to util.dbuffer for buffering incoming data This is primarily a step towards saving uploads directly to files, tho this should hopefully be more efficient than collapsing the entire buffer to a single string every now and then.
author Kim Alvefur <zash@zash.se>
date Sat, 01 Aug 2020 18:14:09 +0200
parent 10580:b7c5d7bae4ef
child 11021:9673c95895fb
comparison
equal deleted inserted replaced
11019:d1604721b665 11020:7076ed654ac9
1 local tonumber = tonumber; 1 local tonumber = tonumber;
2 local assert = assert; 2 local assert = assert;
3 local t_insert, t_concat = table.insert, table.concat;
4 local url_parse = require "socket.url".parse; 3 local url_parse = require "socket.url".parse;
5 local urldecode = require "util.http".urldecode; 4 local urldecode = require "util.http".urldecode;
5 local dbuffer = require "util.dbuffer";
6 6
7 local function preprocess_path(path) 7 local function preprocess_path(path)
8 path = urldecode((path:gsub("//+", "/"))); 8 path = urldecode((path:gsub("//+", "/")));
9 if path:sub(1,1) ~= "/" then 9 if path:sub(1,1) ~= "/" then
10 path = "/"..path; 10 path = "/"..path;
26 local httpstream = {}; 26 local httpstream = {};
27 27
28 function httpstream.new(success_cb, error_cb, parser_type, options_cb) 28 function httpstream.new(success_cb, error_cb, parser_type, options_cb)
29 local client = true; 29 local client = true;
30 if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end 30 if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end
31 local buf, buflen, buftable = {}, 0, true;
32 local bodylimit = tonumber(options_cb and options_cb().body_size_limit) or 10*1024*1024; 31 local bodylimit = tonumber(options_cb and options_cb().body_size_limit) or 10*1024*1024;
32 -- https://stackoverflow.com/a/686243
33 -- Indiviual headers can be up to 16k? What madness?
34 local headlimit = tonumber(options_cb and options_cb().head_size_limit) or 10*1024;
33 local buflimit = tonumber(options_cb and options_cb().buffer_size_limit) or bodylimit * 2; 35 local buflimit = tonumber(options_cb and options_cb().buffer_size_limit) or bodylimit * 2;
34 local chunked, chunk_size, chunk_start; 36 local buffer = dbuffer.new(buflimit);
37 local chunked;
35 local state = nil; 38 local state = nil;
36 local packet; 39 local packet;
37 local len; 40 local len;
38 local have_body; 41 local have_body;
39 local error; 42 local error;
40 return { 43 return {
41 feed = function(_, data) 44 feed = function(_, data)
42 if error then return nil, "parse has failed"; end 45 if error then return nil, "parse has failed"; end
43 if not data then -- EOF 46 if not data then -- EOF
44 if buftable then buf, buftable = t_concat(buf), false; end
45 if state and client and not len then -- reading client body until EOF 47 if state and client and not len then -- reading client body until EOF
46 packet.body = buf; 48 buffer:collapse();
49 packet.body = buffer:read_chunk() or "";
47 success_cb(packet); 50 success_cb(packet);
48 elseif buf ~= "" then -- unexpected EOF 51 state = nil;
52 elseif buffer:length() ~= 0 then -- unexpected EOF
49 error = true; return error_cb("unexpected-eof"); 53 error = true; return error_cb("unexpected-eof");
50 end 54 end
51 return; 55 return;
52 end 56 end
53 if buftable then 57 if not buffer:write(data) then error = true; return error_cb("max-buffer-size-exceeded"); end
54 t_insert(buf, data); 58 while buffer:length() > 0 do
55 else
56 buf = { buf, data };
57 buftable = true;
58 end
59 buflen = buflen + #data;
60 if buflen > buflimit then error = true; return error_cb("max-buffer-size-exceeded"); end
61 while buflen > 0 do
62 if state == nil then -- read request 59 if state == nil then -- read request
63 if buftable then buf, buftable = t_concat(buf), false; end 60 local index = buffer:sub(1, headlimit):find("\r\n\r\n", nil, true);
64 local index = buf:find("\r\n\r\n", nil, true);
65 if not index then return; end -- not enough data 61 if not index then return; end -- not enough data
66 -- FIXME was reason_phrase meant to be passed on somewhere? 62 -- FIXME was reason_phrase meant to be passed on somewhere?
67 local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase 63 local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase
68 local first_line; 64 local first_line;
69 local headers = {}; 65 local headers = {};
70 for line in buf:sub(1,index+1):gmatch("([^\r\n]+)\r\n") do -- parse request 66 for line in buffer:read(index+3):gmatch("([^\r\n]+)\r\n") do -- parse request
71 if first_line then 67 if first_line then
72 local key, val = line:match("^([^%s:]+): *(.*)$"); 68 local key, val = line:match("^([^%s:]+): *(.*)$");
73 if not key then error = true; return error_cb("invalid-header-line"); end -- TODO handle multi-line and invalid headers 69 if not key then error = true; return error_cb("invalid-header-line"); end -- TODO handle multi-line and invalid headers
74 key = key:lower(); 70 key = key:lower();
75 headers[key] = headers[key] and headers[key]..","..val or val; 71 headers[key] = headers[key] and headers[key]..","..val or val;
99 if not have_body then len = 0; end 95 if not have_body then len = 0; end
100 packet = { 96 packet = {
101 code = status_code; 97 code = status_code;
102 httpversion = httpversion; 98 httpversion = httpversion;
103 headers = headers; 99 headers = headers;
104 body = have_body and "" or nil; 100 body = false;
105 -- COMPAT the properties below are deprecated 101 -- COMPAT the properties below are deprecated
106 responseversion = httpversion; 102 responseversion = httpversion;
107 responseheaders = headers; 103 responseheaders = headers;
108 }; 104 };
109 else 105 else
124 method = method; 120 method = method;
125 url = parsed_url; 121 url = parsed_url;
126 path = path; 122 path = path;
127 httpversion = httpversion; 123 httpversion = httpversion;
128 headers = headers; 124 headers = headers;
129 body = nil; 125 body = false;
126 body_sink = nil;
130 }; 127 };
131 end 128 end
132 buf = buf:sub(index + 4); 129 if chunked then
133 buflen = #buf; 130 packet.body_buffer = dbuffer.new(buflimit);
131 end
134 state = true; 132 state = true;
135 end 133 end
136 if state then -- read body 134 if state then -- read body
137 if client then 135 if chunked then
138 if chunked then 136 local chunk_header = buffer:sub(1, 512); -- XXX How large do chunk headers grow?
139 if chunk_start and buflen - chunk_start - 2 < chunk_size then 137 local chunk_size, chunk_start = chunk_header:match("^(%x+)[^\r\n]*\r\n()");
140 return; 138 if not chunk_size then return; end
141 end -- not enough data 139 chunk_size = chunk_size and tonumber(chunk_size, 16);
142 if buftable then buf, buftable = t_concat(buf), false; end 140 if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end
143 if not buf:find("\r\n", nil, true) then 141 if chunk_size == 0 and chunk_header:find("\r\n\r\n", chunk_start-2, true) then
144 return; 142 local body_buffer = packet.body_buffer;
145 end -- not enough data 143 if body_buffer then
146 if not chunk_size then 144 packet.body_buffer = nil;
147 chunk_size, chunk_start = buf:match("^(%x+)[^\r\n]*\r\n()"); 145 body_buffer:collapse();
148 chunk_size = chunk_size and tonumber(chunk_size, 16); 146 packet.body = body_buffer:read_chunk() or "";
149 if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end
150 end 147 end
151 if chunk_size == 0 and buf:find("\r\n\r\n", chunk_start-2, true) then 148
152 state, chunk_size = nil, nil; 149 buffer:collapse();
153 buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped 150 local buf = buffer:read_chunk();
154 success_cb(packet); 151 buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped
155 elseif buflen - chunk_start - 2 >= chunk_size then -- we have a chunk 152 buffer:write(buf);
156 packet.body = packet.body..buf:sub(chunk_start, chunk_start + (chunk_size-1)); 153 state, chunked = nil, nil;
157 buf = buf:sub(chunk_start + chunk_size + 2); 154 success_cb(packet);
158 buflen = buflen - (chunk_start + chunk_size + 2 - 1); 155 elseif buffer:length() - chunk_start - 2 >= chunk_size then -- we have a chunk
159 chunk_size, chunk_start = nil, nil; 156 buffer:discard(chunk_start - 1); -- TODO verify that it's not off-by-one
160 else -- Partial chunk remaining 157 packet.body_buffer:write(buffer:read(chunk_size));
161 break; 158 buffer:discard(2); -- CRLF
162 end 159 else -- Partial chunk remaining
163 elseif len and buflen >= len then
164 if buftable then buf, buftable = t_concat(buf), false; end
165 if packet.code == 101 then
166 packet.body, buf, buflen, buftable = buf, {}, 0, true;
167 else
168 packet.body, buf = buf:sub(1, len), buf:sub(len + 1);
169 buflen = #buf;
170 end
171 state = nil; success_cb(packet);
172 else
173 break; 160 break;
174 end 161 end
175 elseif buflen >= len then 162 elseif buffer:length() >= len then
176 if buftable then buf, buftable = t_concat(buf), false; end 163 assert(not chunked)
177 packet.body, buf = buf:sub(1, len), buf:sub(len + 1); 164 packet.body = buffer:read(len) or "";
178 buflen = #buf;
179 state = nil; success_cb(packet); 165 state = nil; success_cb(packet);
180 else 166 else
181 break; 167 break;
182 end 168 end
169 else
170 break;
183 end 171 end
184 end 172 end
185 end; 173 end;
186 }; 174 };
187 end 175 end