Annotate

plugins/mod_http_file_share.lua @ 11954:b963ac00c967

mod_admin_shell: Remove now redundant promise awaiting in dns:lookup()
author Kim Alvefur <zash@zash.se>
date Sat, 27 Nov 2021 14:16:13 +0100
parent 11873:2b85e4e7d389
child 11990:6f4790b8deec
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
1 -- Prosody IM
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
2 -- Copyright (C) 2021 Kim Alvefur
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
3 --
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
4 -- This project is MIT/X11 licensed. Please see the
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
5 -- COPYING file in the source package for more information.
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
6 --
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
7 -- XEP-0363: HTTP File Upload
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
8 -- Again, from the top!
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
9
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
10 local t_insert = table.insert;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
11 local jid = require "util.jid";
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
12 local st = require "util.stanza";
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
13 local url = require "socket.url";
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
14 local dm = require "core.storagemanager".olddm;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
15 local jwt = require "util.jwt";
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
16 local errors = require "util.error";
11314
7c8b02c5a335 mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents: 11313
diff changeset
17 local dataform = require "util.dataforms".new;
11321
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
18 local dt = require "util.datetime";
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
19 local hi = require "util.human.units";
11332
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
20 local cache = require "util.cache";
11495
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
21 local lfs = require "lfs";
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
22
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
23 local namespace = "urn:xmpp:http:upload:0";
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
24
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
25 module:depends("disco");
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
26
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
27 module:add_identity("store", "file", module:get_option_string("name", "HTTP File Upload"));
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
28 module:add_feature(namespace);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
29
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
30 local uploads = module:open_store("uploads", "archive");
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
31 -- id, <request>, time, owner
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
32
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
33 local secret = module:get_option_string(module.name.."_secret", require"util.id".long());
11310
d1a0f2e918c0 mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents: 11309
diff changeset
34 local external_base_url = module:get_option_string(module.name .. "_base_url");
11314
7c8b02c5a335 mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents: 11313
diff changeset
35 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB
11315
c52fcea39c8e mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents: 11314
diff changeset
36 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {});
11333
f80056b97cf0 mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents: 11332
diff changeset
37 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
38 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
11346
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
39 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
40 local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil);
11310
d1a0f2e918c0 mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents: 11309
diff changeset
41
11311
9edda2026e57 mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents: 11310
diff changeset
42 local access = module:get_option_set(module.name .. "_access", {});
9edda2026e57 mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents: 11310
diff changeset
43
11310
d1a0f2e918c0 mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents: 11309
diff changeset
44 if not external_base_url then
d1a0f2e918c0 mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents: 11309
diff changeset
45 module:depends("http");
d1a0f2e918c0 mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents: 11309
diff changeset
46 end
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
47
11314
7c8b02c5a335 mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents: 11313
diff changeset
48 module:add_extension(dataform {
7c8b02c5a335 mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents: 11313
diff changeset
49 { name = "FORM_TYPE", type = "hidden", value = namespace },
11873
2b85e4e7d389 mod_http_file_share: Move number coercion into util.dataforms
Kim Alvefur <zash@zash.se>
parents: 11865
diff changeset
50 { name = "max-file-size", type = "text-single", datatype = "xs:integer" },
2b85e4e7d389 mod_http_file_share: Move number coercion into util.dataforms
Kim Alvefur <zash@zash.se>
parents: 11865
diff changeset
51 }:form({ ["max-file-size"] = file_size_limit }, "result"));
11314
7c8b02c5a335 mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents: 11313
diff changeset
52
11312
aade4a6179a3 mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents: 11311
diff changeset
53 local upload_errors = errors.init(module.name, namespace, {
11317
79e1f407b6f5 mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents: 11316
diff changeset
54 access = { type = "auth"; condition = "forbidden" };
79e1f407b6f5 mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents: 11316
diff changeset
55 filename = { type = "modify"; condition = "bad-request"; text = "Invalid filename" };
79e1f407b6f5 mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents: 11316
diff changeset
56 filetype = { type = "modify"; condition = "not-acceptable"; text = "File type not allowed" };
79e1f407b6f5 mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents: 11316
diff changeset
57 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large";
11318
3b16aba6285f mod_http_file_share: Fix name of max-file-size tag
Kim Alvefur <zash@zash.se>
parents: 11317
diff changeset
58 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) };
11330
f2c9492e3d25 mod_http_file_share: Fix the obligatory misplaced closing bracket (thanks scansion)
Kim Alvefur <zash@zash.se>
parents: 11329
diff changeset
59 };
11345
0fec04b64a49 mod_http_file_share: Add missing semicolon
Kim Alvefur <zash@zash.se>
parents: 11343
diff changeset
60 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
11346
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
61 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
62 unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
63 outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
11312
aade4a6179a3 mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents: 11311
diff changeset
64 });
aade4a6179a3 mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents: 11311
diff changeset
65
11332
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
66 local upload_cache = cache.new(1024);
11348
f076199b4d38 mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents: 11347
diff changeset
67 local quota_cache = cache.new(1024);
11332
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
68
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
69 local total_storage_usage = nil;
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
70
11491
c3fb802f9e45 mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents: 11406
diff changeset
71 local measure_upload_cache_size = module:measure("upload_cache", "amount");
c3fb802f9e45 mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents: 11406
diff changeset
72 local measure_quota_cache_size = module:measure("quota_cache", "amount");
11794
5d925f340ae6 mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents: 11784
diff changeset
73 local measure_total_storage_usage = nil;
5d925f340ae6 mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents: 11784
diff changeset
74 if total_storage_limit then
5d925f340ae6 mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents: 11784
diff changeset
75 measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" });
5d925f340ae6 mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents: 11784
diff changeset
76 end
11491
c3fb802f9e45 mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents: 11406
diff changeset
77
c3fb802f9e45 mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents: 11406
diff changeset
78 module:hook_global("stats-update", function ()
c3fb802f9e45 mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents: 11406
diff changeset
79 measure_upload_cache_size(upload_cache:count());
c3fb802f9e45 mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents: 11406
diff changeset
80 measure_quota_cache_size(quota_cache:count());
11794
5d925f340ae6 mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents: 11784
diff changeset
81 if total_storage_limit then
5d925f340ae6 mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents: 11784
diff changeset
82 measure_total_storage_usage(total_storage_usage);
5d925f340ae6 mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents: 11784
diff changeset
83 end
11491
c3fb802f9e45 mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents: 11406
diff changeset
84 end);
c3fb802f9e45 mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents: 11406
diff changeset
85
11594
19aac4247b03 mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents: 11568
diff changeset
86 local buckets = {};
19aac4247b03 mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents: 11568
diff changeset
87 for n = 10, 40, 2 do
19aac4247b03 mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents: 11568
diff changeset
88 local exp = math.floor(2 ^ n);
19aac4247b03 mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents: 11568
diff changeset
89 table.insert(buckets, exp);
19aac4247b03 mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents: 11568
diff changeset
90 if exp >= file_size_limit then break end
19aac4247b03 mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents: 11568
diff changeset
91 end
19aac4247b03 mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents: 11568
diff changeset
92 local measure_uploads = module:measure("upload", "sizes", {buckets = buckets});
11355
89efa3f2966b mod_http_file_share: Collect statistics of files uploaded
Kim Alvefur <zash@zash.se>
parents: 11350
diff changeset
93
11321
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
94 -- Convenience wrapper for logging file sizes
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
95 local function B(bytes) return hi.format(bytes, "B", "b"); end
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
96
11325
76fc73d39092 mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents: 11324
diff changeset
97 local function get_filename(slot, create)
76fc73d39092 mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents: 11324
diff changeset
98 return dm.getpath(slot, module.host, module.name, "bin", create)
76fc73d39092 mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents: 11324
diff changeset
99 end
76fc73d39092 mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents: 11324
diff changeset
100
11346
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
101 function get_daily_quota(uploader)
11347
5b3ad3c7fe47 mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents: 11346
diff changeset
102 local now = os.time();
5b3ad3c7fe47 mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents: 11346
diff changeset
103 local max_age = now - 86400;
11348
f076199b4d38 mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents: 11347
diff changeset
104 local cached = quota_cache:get(uploader);
f076199b4d38 mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents: 11347
diff changeset
105 if cached and cached.time > max_age then
f076199b4d38 mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents: 11347
diff changeset
106 return cached.size;
f076199b4d38 mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents: 11347
diff changeset
107 end
11347
5b3ad3c7fe47 mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents: 11346
diff changeset
108 local iter, err = uploads:find(nil, {with = uploader; start = max_age });
11346
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
109 if not iter then return iter, err; end
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
110 local total_bytes = 0;
11349
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
111 local oldest_upload = now;
11348
f076199b4d38 mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents: 11347
diff changeset
112 for _, slot, when in iter do
11346
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
113 local size = tonumber(slot.attr.size);
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
114 if size then total_bytes = total_bytes + size; end
11349
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
115 if when < oldest_upload then oldest_upload = when; end
11346
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
116 end
11349
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
117 -- If there were no uploads then we end up caching [now, 0], which is fine
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
118 -- since we increase the size on new uploads
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
119 quota_cache:set(uploader, { time = oldest_upload, size = total_bytes });
11346
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
120 return total_bytes;
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
121 end
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
122
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
123 function may_upload(uploader, filename, filesize, filetype) -- > boolean, error
11311
9edda2026e57 mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents: 11310
diff changeset
124 local uploader_host = jid.host(uploader);
9edda2026e57 mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents: 11310
diff changeset
125 if not ((access:empty() and prosody.hosts[uploader_host]) or access:contains(uploader) or access:contains(uploader_host)) then
11312
aade4a6179a3 mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents: 11311
diff changeset
126 return false, upload_errors.new("access");
11311
9edda2026e57 mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents: 11310
diff changeset
127 end
9edda2026e57 mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents: 11310
diff changeset
128
11313
e53894d26092 mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents: 11312
diff changeset
129 if not filename or filename:find"/" then
e53894d26092 mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents: 11312
diff changeset
130 -- On Linux, only '/' and '\0' are invalid in filenames and NUL can't be in XML
e53894d26092 mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents: 11312
diff changeset
131 return false, upload_errors.new("filename");
e53894d26092 mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents: 11312
diff changeset
132 end
e53894d26092 mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents: 11312
diff changeset
133
11319
a4b299e37909 mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents: 11318
diff changeset
134 if not filesize or filesize < 0 or filesize % 1 ~= 0 then
a4b299e37909 mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents: 11318
diff changeset
135 return false, upload_errors.new("filesizefmt");
a4b299e37909 mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents: 11318
diff changeset
136 end
11314
7c8b02c5a335 mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents: 11313
diff changeset
137 if filesize > file_size_limit then
7c8b02c5a335 mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents: 11313
diff changeset
138 return false, upload_errors.new("filesize");
7c8b02c5a335 mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents: 11313
diff changeset
139 end
7c8b02c5a335 mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents: 11313
diff changeset
140
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
141 if total_storage_limit then
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
142 if not total_storage_usage then
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
143 return false, upload_errors.new("unknowntotal");
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
144 elseif total_storage_usage + filesize > total_storage_limit then
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
145 module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
146 return false, upload_errors.new("outofdisk");
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
147 end
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
148 end
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
149
11346
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
150 local uploader_quota = get_daily_quota(uploader);
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
151 if uploader_quota + filesize > daily_quota then
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
152 return false, upload_errors.new("quota");
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
153 end
315faec1a920 mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents: 11345
diff changeset
154
11315
c52fcea39c8e mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents: 11314
diff changeset
155 if not ( file_types:empty() or file_types:contains(filetype) or file_types:contains(filetype:gsub("/.*", "/*")) ) then
c52fcea39c8e mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents: 11314
diff changeset
156 return false, upload_errors.new("filetype");
c52fcea39c8e mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents: 11314
diff changeset
157 end
c52fcea39c8e mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents: 11314
diff changeset
158
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
159 return true;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
160 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
161
11350
3287dbdde33e mod_http_file_share: Reorder arguments
Kim Alvefur <zash@zash.se>
parents: 11349
diff changeset
162 function get_authz(slot, uploader, filename, filesize, filetype)
11502
8fd760c04cdf mod_http_file_share: Include time of issuance in auth token
Kim Alvefur <zash@zash.se>
parents: 11501
diff changeset
163 local now = os.time();
11322
4ade9810ce35 mod_http_file_share: Move Authorization type string
Kim Alvefur <zash@zash.se>
parents: 11321
diff changeset
164 return jwt.sign(secret, {
11501
2c9db2278fed mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents: 11500
diff changeset
165 -- token properties
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
166 sub = uploader;
11502
8fd760c04cdf mod_http_file_share: Include time of issuance in auth token
Kim Alvefur <zash@zash.se>
parents: 11501
diff changeset
167 iat = now;
8fd760c04cdf mod_http_file_share: Include time of issuance in auth token
Kim Alvefur <zash@zash.se>
parents: 11501
diff changeset
168 exp = now+300;
11501
2c9db2278fed mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents: 11500
diff changeset
169
2c9db2278fed mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents: 11500
diff changeset
170 -- slot properties
2c9db2278fed mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents: 11500
diff changeset
171 slot = slot;
11503
7adda14945ad mod_http_file_share: Include expiry time of the upload itself in token
Kim Alvefur <zash@zash.se>
parents: 11502
diff changeset
172 expires = expiry >= 0 and (now+expiry) or nil;
11501
2c9db2278fed mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents: 11500
diff changeset
173 -- file properties
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
174 filename = filename;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
175 filesize = filesize;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
176 filetype = filetype;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
177 });
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
178 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
179
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
180 function get_url(slot, filename)
11310
d1a0f2e918c0 mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents: 11309
diff changeset
181 local base_url = external_base_url or module:http_url();
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
182 local slot_url = url.parse(base_url);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
183 slot_url.path = url.parse_path(slot_url.path or "/");
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
184 t_insert(slot_url.path, slot);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
185 if filename then
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
186 t_insert(slot_url.path, filename);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
187 slot_url.path.is_directory = false;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
188 else
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
189 slot_url.path.is_directory = true;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
190 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
191 slot_url.path = url.build_path(slot_url.path);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
192 return url.build(slot_url);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
193 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
194
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
195 function handle_slot_request(event)
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
196 local stanza, origin = event.stanza, event.origin;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
197
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
198 local request = st.clone(stanza.tags[1], true);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
199 local filename = request.attr.filename;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
200 local filesize = tonumber(request.attr.size);
11320
817cadf6be92 mod_http_file_share: Handle content-type being optional
Kim Alvefur <zash@zash.se>
parents: 11319
diff changeset
201 local filetype = request.attr["content-type"] or "application/octet-stream";
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
202 local uploader = jid.bare(stanza.attr.from);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
203
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
204 local may, why_not = may_upload(uploader, filename, filesize, filetype);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
205 if not may then
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
206 origin.send(st.error_reply(stanza, why_not));
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
207 return true;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
208 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
209
11321
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
210 module:log("info", "Issuing upload slot to %s for %s", uploader, B(filesize));
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
211 local slot, storage_err = errors.coerce(uploads:append(nil, nil, request, os.time(), uploader))
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
212 if not slot then
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
213 origin.send(st.error_reply(stanza, storage_err));
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
214 return true;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
215 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
216
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
217 if total_storage_usage then
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
218 total_storage_usage = total_storage_usage + filesize;
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
219 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
220 end
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
221
11349
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
222 local cached_quota = quota_cache:get(uploader);
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
223 if cached_quota and cached_quota.time > os.time()-86400 then
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
224 cached_quota.size = cached_quota.size + filesize;
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
225 quota_cache:set(uploader, cached_quota);
a219001b449d mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents: 11348
diff changeset
226 end
11348
f076199b4d38 mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents: 11347
diff changeset
227
11350
3287dbdde33e mod_http_file_share: Reorder arguments
Kim Alvefur <zash@zash.se>
parents: 11349
diff changeset
228 local authz = get_authz(slot, uploader, filename, filesize, filetype);
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
229 local slot_url = get_url(slot, filename);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
230 local upload_url = slot_url;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
231
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
232 local reply = st.reply(stanza)
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
233 :tag("slot", { xmlns = namespace })
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
234 :tag("get", { url = slot_url }):up()
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
235 :tag("put", { url = upload_url })
11322
4ade9810ce35 mod_http_file_share: Move Authorization type string
Kim Alvefur <zash@zash.se>
parents: 11321
diff changeset
236 :text_tag("header", "Bearer "..authz, {name="Authorization"})
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
237 :reset();
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
238
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
239 origin.send(reply);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
240 return true;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
241 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
242
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
243 function handle_upload(event, path) -- PUT /upload/:slot
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
244 local request = event.request;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
245 local authz = request.headers.authorization;
11326
1ecda954fe97 mod_http_file_share: Strip authorization type prefix a bit earlier
Kim Alvefur <zash@zash.se>
parents: 11325
diff changeset
246 if authz then
1ecda954fe97 mod_http_file_share: Strip authorization type prefix a bit earlier
Kim Alvefur <zash@zash.se>
parents: 11325
diff changeset
247 authz = authz:match("^Bearer (.*)")
1ecda954fe97 mod_http_file_share: Strip authorization type prefix a bit earlier
Kim Alvefur <zash@zash.se>
parents: 11325
diff changeset
248 end
1ecda954fe97 mod_http_file_share: Strip authorization type prefix a bit earlier
Kim Alvefur <zash@zash.se>
parents: 11325
diff changeset
249 if not authz then
11335
b7acab5e7f57 mod_http_file_share: Clarify message about missing Authorization header
Kim Alvefur <zash@zash.se>
parents: 11334
diff changeset
250 module:log("debug", "Missing or malformed Authorization header");
11336
b05331cff47a mod_http_file_share: Indicate missing token via WWW-Authenticate header
Kim Alvefur <zash@zash.se>
parents: 11335
diff changeset
251 event.response.headers.www_authenticate = "Bearer";
11853
ae5ac9830add mod_http_file_share: return 401 instead of 403 if authentication failed
Jonas Schäfer <jonas@wielicki.name>
parents: 11802
diff changeset
252 return 401;
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
253 end
11326
1ecda954fe97 mod_http_file_share: Strip authorization type prefix a bit earlier
Kim Alvefur <zash@zash.se>
parents: 11325
diff changeset
254 local authed, upload_info = jwt.verify(secret, authz);
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
255 if not (authed and type(upload_info) == "table" and type(upload_info.exp) == "number") then
11321
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
256 module:log("debug", "Unauthorized or invalid token: %s, %q", authed, upload_info);
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
257 return 401;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
258 end
11334
dbba2d44fda2 mod_http_file_share: Allow started uploads to complete after token expired
Kim Alvefur <zash@zash.se>
parents: 11333
diff changeset
259 if not request.body_sink and upload_info.exp < os.time() then
11321
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
260 module:log("debug", "Authorization token expired on %s", dt.datetime(upload_info.exp));
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
261 return 410;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
262 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
263 if not path or upload_info.slot ~= path:match("^[^/]+") then
11321
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
264 module:log("debug", "Invalid upload slot: %q, path: %q", upload_info.slot, path);
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
265 return 400;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
266 end
11323
a853a018eede mod_http_file_share: Validate file size early in HTTP PUT request
Kim Alvefur <zash@zash.se>
parents: 11322
diff changeset
267 if request.headers.content_length and tonumber(request.headers.content_length) ~= upload_info.filesize then
a853a018eede mod_http_file_share: Validate file size early in HTTP PUT request
Kim Alvefur <zash@zash.se>
parents: 11322
diff changeset
268 return 413;
a853a018eede mod_http_file_share: Validate file size early in HTTP PUT request
Kim Alvefur <zash@zash.se>
parents: 11322
diff changeset
269 -- Note: We don't know the size if the upload is streamed in chunked encoding,
a853a018eede mod_http_file_share: Validate file size early in HTTP PUT request
Kim Alvefur <zash@zash.se>
parents: 11322
diff changeset
270 -- so we also check the final file size on completion.
a853a018eede mod_http_file_share: Validate file size early in HTTP PUT request
Kim Alvefur <zash@zash.se>
parents: 11322
diff changeset
271 end
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
272
11325
76fc73d39092 mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents: 11324
diff changeset
273 local filename = get_filename(upload_info.slot, true);
11324
494761f5d7da mod_http_file_share: Use '.bin' file extension
Kim Alvefur <zash@zash.se>
parents: 11323
diff changeset
274
11375
6b687210975b mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents: 11374
diff changeset
275 do
6b687210975b mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents: 11374
diff changeset
276 -- check if upload has been completed already
6b687210975b mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents: 11374
diff changeset
277 -- we want to allow retry of a failed upload attempt, but not after it's been completed
6b687210975b mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents: 11374
diff changeset
278 local f = io.open(filename, "r");
6b687210975b mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents: 11374
diff changeset
279 if f then
6b687210975b mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents: 11374
diff changeset
280 f:close();
6b687210975b mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents: 11374
diff changeset
281 return 409;
6b687210975b mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents: 11374
diff changeset
282 end
6b687210975b mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents: 11374
diff changeset
283 end
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
284
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
285 if not request.body_sink then
11321
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
286 module:log("debug", "Preparing to receive upload into %q, expecting %s", filename, B(upload_info.filesize));
11500
21706a581b8a mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents: 11499
diff changeset
287 local fh, err = io.open(filename.."~", "w");
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
288 if not fh then
11500
21706a581b8a mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents: 11499
diff changeset
289 module:log("error", "Could not open file for writing: %s", err);
21706a581b8a mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents: 11499
diff changeset
290 return 500;
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
291 end
11865
77bbbd4263d7 mod_http_file_share: Silence luacheck warning
Kim Alvefur <zash@zash.se>
parents: 11864
diff changeset
292 function event.response:on_destroy() -- luacheck: ignore 212/self
11864
fceebfb28d86 mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents: 11857
diff changeset
293 -- Clean up incomplete upload
fceebfb28d86 mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents: 11857
diff changeset
294 if io.type(fh) == "file" then -- still open
fceebfb28d86 mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents: 11857
diff changeset
295 fh:close();
fceebfb28d86 mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents: 11857
diff changeset
296 os.remove(filename.."~");
fceebfb28d86 mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents: 11857
diff changeset
297 end
fceebfb28d86 mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents: 11857
diff changeset
298 end
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
299 request.body_sink = fh;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
300 if request.body == false then
11374
5b8aec0609f0 mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents: 11357
diff changeset
301 if request.headers.expect == "100-continue" then
5b8aec0609f0 mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents: 11357
diff changeset
302 request.conn:write("HTTP/1.1 100 Continue\r\n\r\n");
5b8aec0609f0 mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents: 11357
diff changeset
303 end
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
304 return true;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
305 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
306 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
307
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
308 if request.body then
11321
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
309 module:log("debug", "Complete upload available, %s", B(#request.body));
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
310 -- Small enough to have been uploaded already
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
311 local written, err = errors.coerce(request.body_sink:write(request.body));
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
312 if not written then
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
313 return err;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
314 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
315 request.body = nil;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
316 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
317
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
318 if request.body_sink then
11316
ae0461b37fbe mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents: 11315
diff changeset
319 local final_size = request.body_sink:seek();
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
320 local uploaded, err = errors.coerce(request.body_sink:close());
11316
ae0461b37fbe mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents: 11315
diff changeset
321 if final_size ~= upload_info.filesize then
ae0461b37fbe mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents: 11315
diff changeset
322 -- Could be too short as well, but we say the same thing
ae0461b37fbe mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents: 11315
diff changeset
323 uploaded, err = false, 413;
ae0461b37fbe mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents: 11315
diff changeset
324 end
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
325 if uploaded then
11321
15ab878a7d23 mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents: 11320
diff changeset
326 module:log("debug", "Upload of %q completed, %s", filename, B(final_size));
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
327 assert(os.rename(filename.."~", filename));
11355
89efa3f2966b mod_http_file_share: Collect statistics of files uploaded
Kim Alvefur <zash@zash.se>
parents: 11350
diff changeset
328 measure_uploads(final_size);
11332
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
329
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
330 upload_cache:set(upload_info.slot, {
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
331 name = upload_info.filename;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
332 size = tostring(upload_info.filesize);
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
333 type = upload_info.filetype;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
334 time = os.time();
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
335 });
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
336 return 201;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
337 else
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
338 assert(os.remove(filename.."~"));
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
339 return err;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
340 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
341 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
342
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
343 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
344
11357
8cb2a64b15da mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents: 11356
diff changeset
345 local download_cache_hit = module:measure("download_cache_hit", "rate");
8cb2a64b15da mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents: 11356
diff changeset
346 local download_cache_miss = module:measure("download_cache_miss", "rate");
8cb2a64b15da mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents: 11356
diff changeset
347
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
348 function handle_download(event, path) -- GET /uploads/:slot+filename
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
349 local request, response = event.request, event.response;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
350 local slot_id = path:match("^[^/]+");
11331
7a915fa49373 mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents: 11330
diff changeset
351 local basename, filetime, filetype, filesize;
11332
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
352 local cached = upload_cache:get(slot_id);
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
353 if cached then
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
354 module:log("debug", "Cache hit");
11357
8cb2a64b15da mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents: 11356
diff changeset
355 download_cache_hit();
11332
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
356 basename = cached.name;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
357 filesize = cached.size;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
358 filetype = cached.type;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
359 filetime = cached.time;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
360 upload_cache:set(slot_id, cached);
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
361 -- TODO cache negative hits?
11331
7a915fa49373 mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents: 11330
diff changeset
362 else
11332
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
363 module:log("debug", "Cache miss");
11357
8cb2a64b15da mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents: 11356
diff changeset
364 download_cache_miss();
11332
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
365 local slot, when = errors.coerce(uploads:get(nil, slot_id));
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
366 if not slot then
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
367 module:log("debug", "uploads:get(%q) --> not-found, %s", slot_id, when);
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
368 else
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
369 module:log("debug", "uploads:get(%q) --> %s, %d", slot_id, slot, when);
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
370 basename = slot.attr.filename;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
371 filesize = slot.attr.size;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
372 filetype = slot.attr["content-type"];
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
373 filetime = when;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
374 upload_cache:set(slot_id, {
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
375 name = basename;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
376 size = slot.attr.size;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
377 type = filetype;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
378 time = when;
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
379 });
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
380 end
11331
7a915fa49373 mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents: 11330
diff changeset
381 end
7a915fa49373 mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents: 11330
diff changeset
382 if not basename then
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
383 return 404;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
384 end
11331
7a915fa49373 mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents: 11330
diff changeset
385 local last_modified = os.date('!%a, %d %b %Y %H:%M:%S GMT', filetime);
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
386 if request.headers.if_modified_since == last_modified then
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
387 return 304;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
388 end
11325
76fc73d39092 mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents: 11324
diff changeset
389 local filename = get_filename(slot_id);
11493
77f2d45799ed mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents: 11491
diff changeset
390 local handle, ferr = io.open(filename);
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
391 if not handle then
11499
a8cbbbb1f165 mod_http_file_share: Fix logging of error opening file
Kim Alvefur <zash@zash.se>
parents: 11496
diff changeset
392 module:log("error", "Could not open file for reading: %s", ferr);
11493
77f2d45799ed mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents: 11491
diff changeset
393 -- This can be because the upload slot wasn't used, or the file disappeared
77f2d45799ed mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents: 11491
diff changeset
394 -- somehow, or permission issues.
77f2d45799ed mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents: 11491
diff changeset
395 return 410;
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
396 end
11333
f80056b97cf0 mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents: 11332
diff changeset
397
11564
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
398 local request_range = request.headers.range;
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
399 local response_range;
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
400 if request_range then
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
401 local range_start, range_end = request_range:match("^bytes=(%d+)%-(%d*)$")
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
402 -- Only support resumption, ie ranges from somewhere in the middle until the end of the file.
11568
d5360307a99d mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents: 11564
diff changeset
403 if (range_start and range_start ~= "0") and (range_end == "" or range_end == filesize) then
d5360307a99d mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents: 11564
diff changeset
404 local pos, size = tonumber(range_start), tonumber(filesize);
d5360307a99d mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents: 11564
diff changeset
405 local new_pos = pos < size and handle:seek("set", pos);
d5360307a99d mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents: 11564
diff changeset
406 if new_pos and new_pos < size then
11564
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
407 response_range = "bytes "..range_start.."-"..filesize.."/"..filesize;
11568
d5360307a99d mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents: 11564
diff changeset
408 filesize = string.format("%d", size-pos);
d5360307a99d mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents: 11564
diff changeset
409 else
d5360307a99d mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents: 11564
diff changeset
410 handle:close();
d5360307a99d mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents: 11564
diff changeset
411 return 416;
11564
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
412 end
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
413 end
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
414 end
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
415
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
416
11402
a3be7b3cf1e1 mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents: 11398
diff changeset
417 if not filetype then
a3be7b3cf1e1 mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents: 11398
diff changeset
418 filetype = "application/octet-stream";
a3be7b3cf1e1 mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents: 11398
diff changeset
419 end
11333
f80056b97cf0 mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents: 11332
diff changeset
420 local disposition = "attachment";
f80056b97cf0 mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents: 11332
diff changeset
421 if safe_types:contains(filetype) or safe_types:contains(filetype:gsub("/.*", "/*")) then
f80056b97cf0 mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents: 11332
diff changeset
422 disposition = "inline";
f80056b97cf0 mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents: 11332
diff changeset
423 end
f80056b97cf0 mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents: 11332
diff changeset
424
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
425 response.headers.last_modified = last_modified;
11331
7a915fa49373 mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents: 11330
diff changeset
426 response.headers.content_length = filesize;
11402
a3be7b3cf1e1 mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents: 11398
diff changeset
427 response.headers.content_type = filetype;
11333
f80056b97cf0 mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents: 11332
diff changeset
428 response.headers.content_disposition = string.format("%s; filename=%q", disposition, basename);
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
429
11564
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
430 if response_range then
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
431 response.status_code = 206;
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
432 response.headers.content_range = response_range;
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
433 end
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
434 response.headers.accept_ranges = "bytes";
60e31c9ece57 mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents: 11503
diff changeset
435
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
436 response.headers.cache_control = "max-age=31556952, immutable";
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
437 response.headers.content_security_policy = "default-src 'none'; frame-ancestors 'none';"
11327
6f2b69469060 mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents: 11326
diff changeset
438 response.headers.strict_transport_security = "max-age=31556952";
6f2b69469060 mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents: 11326
diff changeset
439 response.headers.x_content_type_options = "nosniff";
11611
a6d1131ac833 mod_http_file_share: Update comment about x-frame-options
Kim Alvefur <zash@zash.se>
parents: 11594
diff changeset
440 response.headers.x_frame_options = "DENY"; -- COMPAT IE missing support for CSP frame-ancestors
11327
6f2b69469060 mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents: 11326
diff changeset
441 response.headers.x_xss_protection = "1; mode=block";
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
442
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
443 return response:send_file(handle);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
444 end
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
445
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
446 if expiry >= 0 and not external_base_url then
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
447 -- TODO HTTP DELETE to the external endpoint?
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
448 local array = require "util.array";
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
449 local async = require "util.async";
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
450 local ENOENT = require "util.pposix".ENOENT;
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
451
11329
2a431d3ad8f1 mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents: 11328
diff changeset
452 local function sleep(t)
2a431d3ad8f1 mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents: 11328
diff changeset
453 local wait, done = async.waiter();
2a431d3ad8f1 mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents: 11328
diff changeset
454 module:add_timer(t, done)
2a431d3ad8f1 mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents: 11328
diff changeset
455 wait();
2a431d3ad8f1 mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents: 11328
diff changeset
456 end
2a431d3ad8f1 mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents: 11328
diff changeset
457
11802
3d411720e719 mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents: 11794
diff changeset
458 local prune_start = module:measure("prune", "times");
3d411720e719 mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents: 11794
diff changeset
459
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
460 local reaper_task = async.runner(function(boundary_time)
11802
3d411720e719 mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents: 11794
diff changeset
461 local prune_done = prune_start();
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
462 local iter, total = assert(uploads:find(nil, {["end"] = boundary_time; total = true}));
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
463
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
464 if total == 0 then
11343
f125ac529c22 mod_http_file_share: Clarify log message
Kim Alvefur <zash@zash.se>
parents: 11336
diff changeset
465 module:log("info", "No expired uploaded files to prune");
11356
43e5429ab355 mod_http_file_share: Measure how long it takes to prune expired files
Kim Alvefur <zash@zash.se>
parents: 11355
diff changeset
466 prune_done();
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
467 return;
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
468 end
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
469
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
470 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
11784
f0971a9eba88 mod_http_file_share: Fix traceback in global quota debug logging (thanks Martin)
Kim Alvefur <zash@zash.se>
parents: 11781
diff changeset
471 if total_storage_limit then
f0971a9eba88 mod_http_file_share: Fix traceback in global quota debug logging (thanks Martin)
Kim Alvefur <zash@zash.se>
parents: 11781
diff changeset
472 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
f0971a9eba88 mod_http_file_share: Fix traceback in global quota debug logging (thanks Martin)
Kim Alvefur <zash@zash.se>
parents: 11781
diff changeset
473 end
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
474
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
475 local obsolete_uploads = array();
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
476 local i = 0;
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
477 local size_sum = 0;
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
478 for slot_id, slot_info in iter do
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
479 i = i + 1;
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
480 obsolete_uploads:push(slot_id);
11332
3e0dcdf6283e mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents: 11331
diff changeset
481 upload_cache:set(slot_id, nil);
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
482 size_sum = size_sum + tonumber(slot_info.attr.size);
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
483 end
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
484
11329
2a431d3ad8f1 mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents: 11328
diff changeset
485 sleep(0.1);
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
486 local n = 0;
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
487 local problem_deleting = false;
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
488 obsolete_uploads:filter(function(slot_id)
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
489 n = n + 1;
11329
2a431d3ad8f1 mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents: 11328
diff changeset
490 if i % 100 == 0 then sleep(0.1); end
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
491 local filename = get_filename(slot_id);
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
492 local deleted, err, errno = os.remove(filename);
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
493 if deleted or errno == ENOENT then
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
494 return true;
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
495 else
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
496 module:log("error", "Could not delete file %q: %s", filename, err);
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
497 problem_deleting = true;
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
498 return false;
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
499 end
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
500 end);
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
501 -- obsolete_uploads now contains slot ids for which the files have been
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
502 -- deleted and that needs to be cleared from the database
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
503
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
504 local deletion_query = {["end"] = boundary_time};
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
505 if not problem_deleting then
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
506 module:log("info", "All (%d, %s) expired files successfully deleted", n, B(size_sum));
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
507 if total_storage_usage then
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
508 total_storage_usage = total_storage_usage - size_sum;
11784
f0971a9eba88 mod_http_file_share: Fix traceback in global quota debug logging (thanks Martin)
Kim Alvefur <zash@zash.se>
parents: 11781
diff changeset
509 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
510 end
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
511 -- we can delete based on time
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
512 else
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
513 module:log("warn", "%d out of %d expired files could not be deleted", n-#obsolete_uploads, n);
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
514 -- we'll need to delete only those entries where the files were
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
515 -- successfully deleted, and then try again with the failed ones.
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
516 -- eventually the admin ought to notice and fix the permissions or
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
517 -- whatever the problem is.
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
518 -- total_storage_limit will be inaccurate until this has been resolved
11405
ce8291e89d67 mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents: 11402
diff changeset
519 deletion_query = {ids = obsolete_uploads};
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
520 end
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
521
11406
9d6545a7d483 mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents: 11405
diff changeset
522 if #obsolete_uploads == 0 then
9d6545a7d483 mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents: 11405
diff changeset
523 module:log("debug", "No metadata to remove");
9d6545a7d483 mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents: 11405
diff changeset
524 else
9d6545a7d483 mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents: 11405
diff changeset
525 local removed, err = uploads:delete(nil, deletion_query);
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
526
11406
9d6545a7d483 mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents: 11405
diff changeset
527 if removed == true or removed == n or removed == #obsolete_uploads then
9d6545a7d483 mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents: 11405
diff changeset
528 module:log("debug", "Removed all metadata for expired uploaded files");
9d6545a7d483 mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents: 11405
diff changeset
529 else
9d6545a7d483 mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents: 11405
diff changeset
530 module:log("error", "Problem removing metadata for deleted files: %s", err);
9d6545a7d483 mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents: 11405
diff changeset
531 end
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
532 end
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
533
11356
43e5429ab355 mod_http_file_share: Measure how long it takes to prune expired files
Kim Alvefur <zash@zash.se>
parents: 11355
diff changeset
534 prune_done();
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
535 end);
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
536
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
537 module:add_timer(5, function ()
11328
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
538 reaper_task:run(os.time()-expiry);
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
539 return 60*60;
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
540 end);
ceaa3cebf28b mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents: 11327
diff changeset
541 end
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
542
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
543 if total_storage_limit then
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
544 local async = require "util.async";
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
545
11802
3d411720e719 mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents: 11794
diff changeset
546 local summary_start = module:measure("summary", "times");
3d411720e719 mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents: 11794
diff changeset
547
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
548 local summarizer_task = async.runner(function()
11802
3d411720e719 mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents: 11794
diff changeset
549 local summary_done = summary_start();
11781
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
550 local iter = assert(uploads:find(nil));
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
551
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
552 local count, sum = 0, 0;
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
553 for _, file in iter do
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
554 sum = sum + tonumber(file.attr.size);
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
555 count = count + 1;
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
556 end
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
557
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
558 module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
559 total_storage_usage = sum;
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
560 module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
561 summary_done();
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
562 end);
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
563
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
564 module:add_timer(1, function()
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
565 summarizer_task:run(true);
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
566 return 11 * 60 * 60;
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
567 end);
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
568 end
9c23e7c8a67a mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents: 11611
diff changeset
569
11495
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
570 -- Reachable from the console
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
571 function check_files(query)
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
572 local issues = {};
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
573 local iter = assert(uploads:find(nil, query));
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
574 for slot_id, file in iter do
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
575 local filename = get_filename(slot_id);
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
576 local size, err = lfs.attributes(filename, "size");
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
577 if not size then
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
578 issues[filename] = err;
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
579 elseif tonumber(file.attr.size) ~= size then
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
580 issues[filename] = "file size mismatch";
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
581 end
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
582 end
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
583
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
584 return next(issues) == nil, issues;
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
585 end
6d3f84148729 mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents: 11493
diff changeset
586
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
587 module:hook("iq-get/host/urn:xmpp:http:upload:0:request", handle_slot_request);
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
588
11310
d1a0f2e918c0 mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents: 11309
diff changeset
589 if not external_base_url then
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
590 module:provides("http", {
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
591 streaming_uploads = true;
11398
a1f26d17d70f mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents: 11394
diff changeset
592 cors = {
a1f26d17d70f mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents: 11394
diff changeset
593 credentials = true;
11857
e080d6aa0b3b mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents: 11853
diff changeset
594 headers = {
e080d6aa0b3b mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents: 11853
diff changeset
595 Authorization = true;
e080d6aa0b3b mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents: 11853
diff changeset
596 };
11398
a1f26d17d70f mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents: 11394
diff changeset
597 };
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
598 route = {
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
599 ["PUT /*"] = handle_upload;
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
600 ["GET /*"] = handle_download;
11394
420787340339 mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents: 11375
diff changeset
601 ["GET /"] = function (event)
420787340339 mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents: 11375
diff changeset
602 return prosody.events.fire_event("http-message", {
420787340339 mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents: 11375
diff changeset
603 response = event.response;
420787340339 mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents: 11375
diff changeset
604 ---
420787340339 mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents: 11375
diff changeset
605 title = "Prosody HTTP Upload endpoint";
420787340339 mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents: 11375
diff changeset
606 message = "This is where files will be uploaded to, and served from.";
420787340339 mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents: 11375
diff changeset
607 warning = not (event.request.secure) and "This endpoint is not considered secure!" or nil;
420787340339 mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents: 11375
diff changeset
608 }) or "This is the Prosody HTTP Upload endpoint.";
420787340339 mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents: 11375
diff changeset
609 end
11309
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
610 }
b59aed75dc5e mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff changeset
611 });
11310
d1a0f2e918c0 mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents: 11309
diff changeset
612 end