prosody-modules
d6fb9f7afaa5
mod_log_ringbuffer/mod_log_ringbuffer.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody-modules

Annotate

mod_log_ringbuffer/mod_log_ringbuffer.lua @ 4225:d6fb9f7afaa5

mod_log_ringbuffer: Discard old data when buffer is full
author Matthew Wild <mwild1@gmail.com>
date Tue, 20 Oct 2020 15:34:29 +0100
parent 4219:b3dd77f2d0d7
child 4226:df2ccb42a241
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
4205
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
1 module:set_global();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
2
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
3 local loggingmanager = require "core.loggingmanager";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
4 local format = require "util.format".format;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
5 local pposix = require "util.pposix";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
6 local rb = require "util.ringbuffer";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
7
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
8 local default_timestamp = "%b %d %H:%M:%S ";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
9 local max_chunk_size = module:get_option_number("log_ringbuffer_chunk_size", 16384);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
10
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
11 local os_date = os.date;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
12
4215
86f8ece24029 mod_log_ringbuffer: Update default filename to include data path
Matthew Wild <mwild1@gmail.com>
parents: 4205
diff changeset
13 local default_filename_template = "{paths.data}/ringbuffer-logs-{pid}-{count}.log";
4205
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
14 local render_filename = require "util.interpolation".new("%b{}", function (s) return s; end, {
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
15 yyyymmdd = function (t)
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
16 return os_date("%Y%m%d", t);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
17 end;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
18 hhmmss = function (t)
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
19 return os_date("%H%M%S", t);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
20 end;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
21 });
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
22
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
23 local dump_count = 0;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
24
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
25 local function dump_buffer(buffer, filename)
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
26 dump_count = dump_count + 1;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
27 local f, err = io.open(filename, "a+");
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
28 if not f then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
29 module:log("error", "Unable to open output file: %s", err);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
30 return;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
31 end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
32 local bytes_remaining = buffer:length();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
33 f:write(("-- Dumping %d bytes at %s --\n"):format(bytes_remaining, os_date(default_timestamp)));
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
34 while bytes_remaining > 0 do
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
35 local chunk_size = math.min(bytes_remaining, max_chunk_size);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
36 local chunk = buffer:read(chunk_size);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
37 if not chunk then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
38 f:write("-- Dump aborted due to error --\n\n");
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
39 f:close();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
40 return;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
41 end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
42 f:write(chunk);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
43 bytes_remaining = bytes_remaining - chunk_size;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
44 end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
45 f:write("-- End of dump --\n\n");
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
46 f:close();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
47 end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
48
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
49 local function get_filename(filename_template)
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
50 filename_template = filename_template or default_filename_template;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
51 return render_filename(filename_template, {
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
52 paths = prosody.paths;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
53 pid = pposix.getpid();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
54 count = dump_count;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
55 time = os.time();
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
56 });
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
57 end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
58
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
59 local function ringbuffer_log_sink_maker(sink_config)
4225
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
60 local buffer_size = sink_config.size or 100*1024;
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
61 local buffer = rb.new(buffer_size);
4205
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
62
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
63 local timestamps = sink_config.timestamps;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
64
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
65 if timestamps == true or timestamps == nil then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
66 timestamps = default_timestamp; -- Default format
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
67 elseif timestamps then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
68 timestamps = timestamps .. " ";
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
69 end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
70
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
71 local function dump()
4219
b3dd77f2d0d7 mod_log_ringbuffer: Switch `filename` to not be interpolated, add filename_template which is
Matthew Wild <mwild1@gmail.com>
parents: 4218
diff changeset
72 dump_buffer(buffer, sink_config.filename or get_filename(sink_config.filename_template));
4205
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
73 end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
74
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
75 if sink_config.signal then
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
76 require "util.signal".signal(sink_config.signal, dump);
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
77 elseif sink_config.event then
4218
f917bb78ab67 mod_log_ringbuffer: Fix incorrect parameter name
Matthew Wild <mwild1@gmail.com>
parents: 4215
diff changeset
78 module:hook_global(sink_config.event, dump);
4205
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
79 end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
80
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
81 return function (name, level, message, ...)
4225
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
82 local line = format("%s%s\t%s\t%s\n", timestamps and os_date(timestamps) or "", name, level, format(message, ...));
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
83 if not buffer:write(line) then
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
84 if #line > buffer_size then
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
85 buffer:discard(buffer_size);
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
86 buffer:write(line:sub(-buffer_size));
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
87 else
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
88 buffer:discard(#line);
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
89 buffer:write(line);
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
90 end
d6fb9f7afaa5 mod_log_ringbuffer: Discard old data when buffer is full
Matthew Wild <mwild1@gmail.com>
parents: 4219
diff changeset
91 end
4205
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
92 end;
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
93 end
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
94
481c4d75e77d mod_log_ringbuffer: New module to send logs to an in-memory ringbuffer
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
95 loggingmanager.register_sink_type("ringbuffer", ringbuffer_log_sink_maker);