prosody-modules
66986f5271c3
mod_storage_s3/mod_storage_s3.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody-modules

Annotate

mod_storage_s3/mod_storage_s3.lua @ 5719:66986f5271c3

mod_storage_s3: Skip archive items matching on date but not full datetime Since it only encodes dates in paths, it would have returned items from outside the specified start..end range if they were from earlier or later in the same (UTC) day.
author Kim Alvefur <zash@zash.se>
date Sat, 11 Nov 2023 22:26:39 +0100
parent 5718:b4632d5f840b
child 5720:4a0279c5c7ed
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
1 local http = require "prosody.net.http";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
2 local array = require "prosody.util.array";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
3 local async = require "prosody.util.async";
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
4 local dt = require "prosody.util.datetime";
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
5 local hashes = require "prosody.util.hashes";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
6 local httputil = require "prosody.util.http";
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
7 local uuid = require "prosody.util.uuid";
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
8 local it = require "prosody.util.iterators";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9 local jid = require "prosody.util.jid";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10 local json = require "prosody.util.json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
11 local st = require "prosody.util.stanza";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
12 local xml = require "prosody.util.xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
13 local url = require "socket.url";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
14
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
15 local new_uuid = uuid.v7 or uuid.generate;
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
16 local hmac_sha256 = hashes.hmac_sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
17 local sha256 = hashes.sha256;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
18
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
19 local driver = {};
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
20
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
21 local bucket = module:get_option_string("s3_bucket", "prosody");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22 local base_uri = module:get_option_string("s3_base_uri", "http://localhost:9000");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
23 local region = module:get_option_string("s3_region", "us-east-1");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
24
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
25 local access_key = module:get_option_string("s3_access_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
26 local secret_key = module:get_option_string("s3_secret_key");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
27
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
28 local aws4_format = "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
29
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
30 local function aws_auth(event)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
31 local request, options = event.request, event.options;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
32 local method = options.method or "GET";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
33 local query = options.query;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
34 local payload = options.body;
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
35
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
36 local payload_type = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
37 if st.is_stanza(payload) then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
38 payload_type = "application/xml";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
39 payload = tostring(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
40 elseif payload ~= nil then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
41 payload_type = "application/json";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
42 payload = json.encode(payload);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
43 end
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
44 options.body = payload;
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
45
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
46 local payload_hash = sha256(payload or "", true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
47
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
48 local now = os.time();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
49 local aws_datetime = os.date("!%Y%m%dT%H%M%SZ", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
50 local aws_date = os.date("!%Y%m%d", now);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
51
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
52 local headers = {
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
53 ["Accept"] = "*/*";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
54 ["Authorization"] = nil;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
55 ["Content-Type"] = payload_type;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
56 ["Host"] = request.authority;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
57 ["User-Agent"] = "Prosody XMPP Server";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
58 ["X-Amz-Content-Sha256"] = payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
59 ["X-Amz-Date"] = aws_datetime;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
60 };
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
61
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
62 local canonical_uri = url.build({ path = request.path });
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
63 local canonical_query = "";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
64 local canonical_headers = array();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
65 local signed_headers = array()
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
66
5693
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
67 if query then
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
68 local sorted_query = array();
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
69 for name, value in it.sorted_pairs(query) do
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
70 sorted_query:push({ name = name; value = value });
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
71 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
72 sorted_query:sort(function (a,b) return a.name < b.name end)
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
73 canonical_query = httputil.formencode(sorted_query):gsub("%%%x%x", string.upper);
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
74 request.query = canonical_query;
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
75 end
2c9d72ef829e mod_storage_s3: Handle signing of request ?query part
Kim Alvefur <zash@zash.se>
parents: 5692
diff changeset
76
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
77 for header_name, header_value in it.sorted_pairs(headers) do
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
78 header_name = header_name:lower();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
79 canonical_headers:push(header_name .. ":" .. header_value .. "\n");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
80 signed_headers:push(header_name);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
81 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
82
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
83 canonical_headers = canonical_headers:concat();
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
84 signed_headers = signed_headers:concat(";");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
85
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
86 local scope = aws_date .. "/" .. region .. "/s3/aws4_request";
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
87
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
88 local canonical_request = method .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
89 .. canonical_uri .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
90 .. canonical_query .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
91 .. canonical_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
92 .. signed_headers .. "\n"
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
93 .. payload_hash;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
94
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
95 local signature_payload = "AWS4-HMAC-SHA256" .. "\n" .. aws_datetime .. "\n" .. scope .. "\n" .. sha256(canonical_request, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
96
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
97 -- This can be cached?
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
98 local date_key = hmac_sha256("AWS4" .. secret_key, aws_date);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
99 local date_region_key = hmac_sha256(date_key, region);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
100 local date_region_service_key = hmac_sha256(date_region_key, "s3");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
101 local signing_key = hmac_sha256(date_region_service_key, "aws4_request");
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
102
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
103 local signature = hmac_sha256(signing_key, signature_payload, true);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
104
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
105 headers["Authorization"] = string.format(aws4_format, access_key, scope, signed_headers, signature);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
106
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
107 options.headers = headers;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
108 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
109
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
110 function driver:open(store, typ)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
111 local mt = self[typ or "keyval"]
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
112 if not mt then
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
113 return nil, "unsupported-store";
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
114 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
115 local httpclient = http.new({});
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
116 httpclient.events.add_handler("pre-request", aws_auth);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
117 return setmetatable({ store = store; bucket = bucket; type = typ; http = httpclient }, mt);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
118 end
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
119
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
120 local keyval = { };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
121 driver.keyval = { __index = keyval; __name = module.name .. " keyval store" };
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
122
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
123 local function new_request(self, method, path, query, payload)
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
124 local request = url.parse(base_uri);
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
125 request.path = path;
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
126
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
127 return self.http:request(url.build(request), { method = method; body = payload; query = query });
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
128 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
129
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
130 -- coerce result back into Prosody data type
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
131 local function on_result(response)
5699
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents: 5698
diff changeset
132 if response.code == 404 and response.request.method == "GET" then
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents: 5698
diff changeset
133 return nil;
799f69a5921a mod_storage_s3: Treat 404 to GET as a signal for empty data
Kim Alvefur <zash@zash.se>
parents: 5698
diff changeset
134 end
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
135 if response.code >= 400 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
136 error(response.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
137 end
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
138 local content_type = response.headers["content-type"];
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
139 if content_type == "application/json" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
140 return json.decode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
141 elseif content_type == "application/xml" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
142 return xml.parse(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
143 elseif content_type == "application/x-www-form-urlencoded" then
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
144 return httputil.formdecode(response.body);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
145 else
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
146 response.log("warn", "Unknown response data type %s", content_type);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
147 return response.body;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
148 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
149 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
150
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
151 function keyval:_path(key)
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
152 return url.build_path({
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
153 is_absolute = true;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
154 bucket;
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
155 jid.escape(module.host);
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
156 jid.escape(self.store);
5698
17ea26cf7259 mod_storage_s3: Use '@' as placeholder for empty (host) store slots
Kim Alvefur <zash@zash.se>
parents: 5697
diff changeset
157 jid.escape(key or "@");
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
158 })
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
159 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
160
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
161 function keyval:get(user)
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
162 return async.wait_for(new_request(self, "GET", self:_path(user)):next(on_result));
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
163 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
164
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
165 function keyval:set(user, data)
5694
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5693
diff changeset
166
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5693
diff changeset
167 if data == nil or (type(data) == "table" and next(data) == nil) then
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
168 return async.wait_for(new_request(self, "DELETE", self:_path(user)));
5694
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5693
diff changeset
169 end
c8322c64a548 mod_storage_s3: Implement keyvalue deletion
Kim Alvefur <zash@zash.se>
parents: 5693
diff changeset
170
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
171 return async.wait_for(new_request(self, "PUT", self:_path(user), nil, data));
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
172 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
173
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
174 function keyval:users()
5695
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
175 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
176 local prefix = url.build_path({ jid.escape(module.host); jid.escape(self.store); is_directory = true });
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
177 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, { prefix = prefix }))
5695
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
178 if err or list_result.code ~= 200 then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
179 return nil, err;
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
180 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
181 local list_bucket_result = xml.parse(list_result.body);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
182 if list_bucket_result:get_child_text("IsTruncated") == "true" then
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
183 local max_keys = list_bucket_result:get_child_text("MaxKeys");
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
184 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
185 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
186 local keys = array();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
187 for content in list_bucket_result:childtags("Contents") do
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
188 local key = url.parse_path(content:get_child_text("Key"));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
189 keys:push(jid.unescape(key[3]));
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
190 end
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
191 return function()
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
192 return keys:pop();
c74a96dc5d58 mod_storage_s3: Implement iteration of keyvalue keys (users usually)
Kim Alvefur <zash@zash.se>
parents: 5694
diff changeset
193 end
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
194 end
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
195
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
196 local archive = {};
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
197 driver.archive = { __index = archive };
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
198
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
199 archive.caps = {
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
200 };
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
201
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
202 function archive:_path(username, date, when, with, key)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
203 return url.build_path({
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
204 is_absolute = true;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
205 bucket;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
206 jid.escape(module.host);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
207 jid.escape(self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
208 jid.escape(username);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
209 jid.escape(jid.prep(with));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
210 date or dt.date(when);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
211 key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
212 })
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
213 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
214
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
215
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
216 -- PUT .../with/when/id
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
217 function archive:append(username, key, value, when, with)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
218 local wrapper = st.stanza("wrapper");
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
219 -- Minio had trouble with timestamps, probably the ':' characters, in paths.
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
220 wrapper:tag("delay", { xmlns = "urn:xmpp:delay"; stamp = dt.datetime(when) }):up();
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
221 wrapper:add_direct_child(value);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
222 key = key or new_uuid();
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
223 return async.wait_for(new_request(self, "PUT", self:_path(username, nil, when, with, key), nil, wrapper):next(function(r)
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
224 if r.code == 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
225 return key;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
226 else
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
227 error(r.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
228 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
229 end));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
230 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
231
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
232 function archive:find(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
233 local bucket_path = url.build_path({ is_absolute = true; bucket; is_directory = true });
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
234 local prefix = { jid.escape(module.host); jid.escape(self.store); is_directory = true };
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
235 table.insert(prefix, jid.escape(username or "@"));
5697
51d0311747fa mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents: 5696
diff changeset
236 if not query then
51d0311747fa mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents: 5696
diff changeset
237 query = {};
51d0311747fa mod_storage_s3: Handle archive query without parameters
Kim Alvefur <zash@zash.se>
parents: 5696
diff changeset
238 end
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
239 if query["with"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
240 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
241 if query["start"] and query["end"] and dt.date(query["start"]) == dt.date(query["end"]) then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
242 table.insert(prefix, sha256(jid.prep(query["with"]), true):sub(1,24));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
243 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
244 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
245
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
246 prefix = url.build_path(prefix);
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
247 local list_result, err = async.wait_for(new_request(self, "GET", bucket_path, {
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
248 prefix = prefix;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
249 ["max-keys"] = query["max"] and tostring(query["max"]);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
250 }));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
251 if err or list_result.code ~= 200 then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
252 return nil, err;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
253 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
254 local list_bucket_result = xml.parse(list_result.body);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
255 if list_bucket_result:get_child_text("IsTruncated") == "true" then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
256 local max_keys = list_bucket_result:get_child_text("MaxKeys");
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
257 module:log("warn", "Paging truncated results not implemented, max %s %s returned", max_keys, self.store);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
258 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
259 local keys = array();
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
260 local iterwrap = function(...)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
261 return ...;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
262 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
263 if query["reverse"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
264 query["before"], query["after"] = query["after"], query["before"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
265 iterwrap = it.reverse;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
266 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
267 local found = not query["after"];
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
268 for content in iterwrap(list_bucket_result:childtags("Contents")) do
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
269 local key = url.parse_path(content:get_child_text("Key"));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
270 if found and query["before"] == key[6] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
271 break
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
272 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
273 if (not query["with"] or query["with"] == jid.unescape(key[5]))
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
274 and (not query["start"] or dt.date(query["start"]) >= key[6])
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
275 and (not query["end"] or dt.date(query["end"]) <= key[6])
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
276 and found then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
277 keys:push({ key = key[6]; date = key[5]; with = jid.unescape(key[4]) });
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
278 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
279 if not found and key[6] == query["after"] then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
280 found = not found
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
281 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
282 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
283 local i = 0;
5719
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5718
diff changeset
284 local function get_next()
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
285 i = i + 1;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
286 local item = keys[i];
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
287 if item == nil then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
288 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
289 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
290 -- luacheck: ignore 431/err
5718
b4632d5f840b mod_storage_s3: Move request signing into a net.http hook
Kim Alvefur <zash@zash.se>
parents: 5699
diff changeset
291 local value, err = async.wait_for(new_request(self, "GET", self:_path(username or "@", item.date, nil, item.with, item.key)):next(on_result));
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
292 if not value then
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
293 module:log("error", "%s", err);
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
294 return nil;
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
295 end
5719
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5718
diff changeset
296 local when = dt.parse(value:get_child_attr("delay", "urn:xmpp:delay", "stamp"));
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
297
5719
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5718
diff changeset
298 if (not query["start"] or query["start"] >= when) and (not query["end"] or query["end"] <= when) then
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5718
diff changeset
299 return item.key, value.tags[2], when, item.with;
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5718
diff changeset
300 else
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5718
diff changeset
301 -- date was correct but not the time
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5718
diff changeset
302 return get_next();
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5718
diff changeset
303 end
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
304 end
5719
66986f5271c3 mod_storage_s3: Skip archive items matching on date but not full datetime
Kim Alvefur <zash@zash.se>
parents: 5718
diff changeset
305 return get_next;
5696
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
306 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
307
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
308 function archive:users()
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
309 return it.unique(keyval.users(self));
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
310 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
311
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
312 --[[ TODO
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
313 function archive:delete(username, query)
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
314 return nil, "not-implemented";
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
315 end
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
316 --]]
b17ba149b7c5 mod_storage_s3: Implement Archive storage
Kim Alvefur <zash@zash.se>
parents: 5695
diff changeset
317
5692
30f91daa40b4 mod_storage_s3: Beginnings of an experimental S3 storage driver
Kim Alvefur <zash@zash.se>
parents:
diff changeset
318 module:provides("storage", driver);