prosody
39991e40d1dc
util/dbuffer.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Comparison

util/dbuffer.lua @ 10973:39991e40d1dc

util.dbuffer: dynamic string buffer Similar to util.ringbuffer (and shares almost identical API). Differences: - size limit is optional and dynamic - does not allocate a fixed buffer of max_size bytes - focus on simply storing references to existing string objects where possible, avoiding unnecessary allocations - references are still stored in a ring buffer to enable use as a fast FIFO Optional second parameter to new() provides the number of ring buffer segments. On Lua 5.2 on my laptop, a segment is ~19 bytes. If the ring buffer fills up, the next write will compact all strings into a single item.
author Matthew Wild <mwild1@gmail.com>
date Fri, 26 Jun 2020 16:41:31 +0100
child 10980:eaee72c7afbd
comparison
equal deleted inserted replaced
10972:b3773b1b90a1 10973:39991e40d1dc
1 local queue = require "util.queue";
2
3 local dbuffer_methods = {};
4 local dynamic_buffer_mt = { __index = dbuffer_methods };
5
6 function dbuffer_methods:write(data)
7 if self.max_size and #data + self._length > self.max_size then
8 return nil;
9 end
10 local ok = self.items:push(data);
11 if not ok then
12 self:collapse();
13 ok = self.items:push(data);
14 end
15 if not ok then
16 return nil;
17 end
18 self._length = self._length + #data;
19 return true;
20 end
21
22 function dbuffer_methods:read_chunk(requested_bytes)
23 local chunk, consumed = self.items:peek(), self.front_consumed;
24 if not chunk then return; end
25 local chunk_length = #chunk;
26 local remaining_chunk_length = chunk_length - consumed;
27 if remaining_chunk_length <= requested_bytes then
28 self.front_consumed = 0;
29 self._length = self._length - remaining_chunk_length;
30 self.items:pop();
31 assert(#chunk:sub(consumed + 1, -1) == remaining_chunk_length);
32 return chunk:sub(consumed + 1, -1), remaining_chunk_length;
33 end
34 local end_pos = consumed + requested_bytes;
35 self.front_consumed = end_pos;
36 self._length = self._length - requested_bytes;
37 assert(#chunk:sub(consumed + 1, end_pos) == requested_bytes);
38 return chunk:sub(consumed + 1, end_pos), requested_bytes;
39 end
40
41 function dbuffer_methods:read(requested_bytes)
42 local chunks;
43
44 if requested_bytes > self._length then
45 return nil;
46 end
47
48 local chunk, read_bytes = self:read_chunk(requested_bytes);
49 if chunk then
50 requested_bytes = requested_bytes - read_bytes;
51 if requested_bytes == 0 then -- Already read everything we need
52 return chunk;
53 end
54 chunks = {};
55 else
56 return nil;
57 end
58
59 -- Need to keep reading more chunks
60 while chunk do
61 table.insert(chunks, chunk);
62 if requested_bytes > 0 then
63 chunk, read_bytes = self:read_chunk(requested_bytes);
64 requested_bytes = requested_bytes - read_bytes;
65 else
66 break;
67 end
68 end
69
70 return table.concat(chunks);
71 end
72
73 function dbuffer_methods:discard(requested_bytes)
74 if requested_bytes > self._length then
75 return nil;
76 end
77
78 local chunk, read_bytes = self:read_chunk(requested_bytes);
79 if chunk then
80 requested_bytes = requested_bytes - read_bytes;
81 if requested_bytes == 0 then -- Already read everything we need
82 return true;
83 end
84 else
85 return nil;
86 end
87
88 while chunk do
89 if requested_bytes > 0 then
90 chunk, read_bytes = self:read_chunk(requested_bytes);
91 requested_bytes = requested_bytes - read_bytes;
92 else
93 break;
94 end
95 end
96 return true;
97 end
98
99 function dbuffer_methods:sub(i, j)
100 if j == nil then
101 j = -1;
102 end
103 if j < 0 then
104 j = self._length + (j+1);
105 end
106 if i < 0 then
107 i = self._length + (i+1);
108 end
109 if i < 1 then
110 i = 1;
111 end
112 if j > self._length then
113 j = self._length;
114 end
115 if i > j then
116 return "";
117 end
118
119 self:collapse(j);
120
121 return self.items:peek():sub(i, j);
122 end
123
124 function dbuffer_methods:byte(i, j)
125 i = i or 1;
126 j = j or i;
127 return string.byte(self:sub(i, j), 1, -1);
128 end
129
130 function dbuffer_methods:length()
131 return self._length;
132 end
133 dynamic_buffer_mt.__len = dbuffer_methods.length; -- support # operator
134
135 function dbuffer_methods:collapse(bytes)
136 bytes = bytes or self._length;
137
138 local front_chunk = self.items:peek();
139
140 if #front_chunk - self.front_consumed >= bytes then
141 return;
142 end
143
144 local front_chunks = { front_chunk:sub(self.front_consumed+1) };
145 local front_bytes = #front_chunks[1];
146
147 while front_bytes < bytes do
148 self.items:pop();
149 local chunk = self.items:peek();
150 front_bytes = front_bytes + #chunk;
151 table.insert(front_chunks, chunk);
152 end
153 self.items:replace(table.concat(front_chunks));
154 self.front_consumed = 0;
155 end
156
157 local function new(max_size, max_chunks)
158 if max_size and max_size <= 0 then
159 return nil;
160 end
161 return setmetatable({
162 front_consumed = 0;
163 _length = 0;
164 max_size = max_size;
165 items = queue.new(max_chunks or 32);
166 }, dynamic_buffer_mt);
167 end
168
169 return {
170 new = new;
171 };