Software /
code /
prosody
Annotate
plugins/mod_http_file_share.lua @ 13114:025c38ee885d
util.dependencies: Print tables itself to reduce number of imports
Rationale: See diffstat
When this module is imported, it ends up calling stty via term_width()
in util.human.io.table(). When this happens outside of a terminal, the
following message is sent to stdout:
stty: 'standard input': Inappropriate ioctl for device
Not importing this module avoids that.
Furthermore three is value in this module having minimal dependencies as
they might not be available when it does the checks.
Ref a1fed82c44b9
author | Kim Alvefur <zash@zash.se> |
---|---|
date | Fri, 26 May 2023 21:18:27 +0200 |
parent | 13056:c38b1c63aa5c |
child | 13168:536055476912 |
rev | line source |
---|---|
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 -- Prosody IM |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 -- Copyright (C) 2021 Kim Alvefur |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
3 -- |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
4 -- This project is MIT/X11 licensed. Please see the |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 -- COPYING file in the source package for more information. |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
6 -- |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
7 -- XEP-0363: HTTP File Upload |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
8 -- Again, from the top! |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
9 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
10 local t_insert = table.insert; |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
11 local jid = require "prosody.util.jid"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
12 local st = require "prosody.util.stanza"; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
13 local url = require "socket.url"; |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
14 local dm = require "prosody.core.storagemanager".olddm; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
15 local errors = require "prosody.util.error"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
16 local dataform = require "prosody.util.dataforms".new; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
17 local urlencode = require "prosody.util.http".urlencode; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
18 local dt = require "prosody.util.datetime"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
19 local hi = require "prosody.util.human.units"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
20 local cache = require "prosody.util.cache"; |
11495
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
21 local lfs = require "lfs"; |
13056
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
22 local parse_duration = require "prosody.util.human.io".parse_duration; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
23 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
24 local unknown = math.abs(0/0); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
25 local unlimited = math.huge; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
26 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
27 local namespace = "urn:xmpp:http:upload:0"; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
28 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
29 module:depends("disco"); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
30 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
31 module:add_identity("store", "file", module:get_option_string("name", "HTTP File Upload")); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
32 module:add_feature(namespace); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
33 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
34 local uploads = module:open_store("uploads", "archive"); |
11999
9d2eab56f124
mod_http_file_share: Keep track of total storage use across restarts
Kim Alvefur <zash@zash.se>
parents:
11998
diff
changeset
|
35 local persist_stats = module:open_store("upload_stats", "map"); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
36 -- id, <request>, time, owner |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
37 |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
38 local secret = module:get_option_string(module.name.."_secret", require"prosody.util.id".long()); |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
39 local external_base_url = module:get_option_string(module.name .. "_base_url"); |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
40 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB |
11315
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
41 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {}); |
11333
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
42 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"}); |
13056
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
43 local expiry_str = module:get_option_string(module.name .. "_expires_after", "1w"); |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
44 local expiry, parse_err = parse_duration(expiry_str); |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
45 if expiry == nil then |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
46 module:log("error", "Could not parse "..module.name.."_expire_after string %q: %s", expiry_str, parse_err); |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
47 return false; |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
48 end |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
49 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
50 local total_storage_limit = module:get_option_number(module.name.."_global_quota", unlimited); |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
51 |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
52 local create_jwt, verify_jwt = require "prosody.util.jwt".init("HS256", secret); |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
53 |
11311
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
54 local access = module:get_option_set(module.name .. "_access", {}); |
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
55 |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
56 if not external_base_url then |
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
57 module:depends("http"); |
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
58 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
59 |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
60 module:add_extension(dataform { |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
61 { name = "FORM_TYPE", type = "hidden", value = namespace }, |
11873
2b85e4e7d389
mod_http_file_share: Move number coercion into util.dataforms
Kim Alvefur <zash@zash.se>
parents:
11865
diff
changeset
|
62 { name = "max-file-size", type = "text-single", datatype = "xs:integer" }, |
2b85e4e7d389
mod_http_file_share: Move number coercion into util.dataforms
Kim Alvefur <zash@zash.se>
parents:
11865
diff
changeset
|
63 }:form({ ["max-file-size"] = file_size_limit }, "result")); |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
64 |
11312
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
65 local upload_errors = errors.init(module.name, namespace, { |
11317
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
66 access = { type = "auth"; condition = "forbidden" }; |
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
67 filename = { type = "modify"; condition = "bad-request"; text = "Invalid filename" }; |
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
68 filetype = { type = "modify"; condition = "not-acceptable"; text = "File type not allowed" }; |
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
69 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large"; |
11318
3b16aba6285f
mod_http_file_share: Fix name of max-file-size tag
Kim Alvefur <zash@zash.se>
parents:
11317
diff
changeset
|
70 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) }; |
11330
f2c9492e3d25
mod_http_file_share: Fix the obligatory misplaced closing bracket (thanks scansion)
Kim Alvefur <zash@zash.se>
parents:
11329
diff
changeset
|
71 }; |
11345
0fec04b64a49
mod_http_file_share: Add missing semicolon
Kim Alvefur <zash@zash.se>
parents:
11343
diff
changeset
|
72 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; }; |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
73 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; }; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
74 outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" }; |
11312
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
75 }); |
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
76 |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
77 local upload_cache = cache.new(1024); |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
78 local quota_cache = cache.new(1024); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
79 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
80 local total_storage_usage = unknown; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
81 |
11491
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
82 local measure_upload_cache_size = module:measure("upload_cache", "amount"); |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
83 local measure_quota_cache_size = module:measure("quota_cache", "amount"); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
84 local measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" }); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
85 |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
86 do |
11999
9d2eab56f124
mod_http_file_share: Keep track of total storage use across restarts
Kim Alvefur <zash@zash.se>
parents:
11998
diff
changeset
|
87 local total, err = persist_stats:get(nil, "total"); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
88 if not err then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
89 total_storage_usage = tonumber(total) or 0; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
90 end |
11794
5d925f340ae6
mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents:
11784
diff
changeset
|
91 end |
11491
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
92 |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
93 module:hook_global("stats-update", function () |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
94 measure_upload_cache_size(upload_cache:count()); |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
95 measure_quota_cache_size(quota_cache:count()); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
96 measure_total_storage_usage(total_storage_usage); |
11491
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
97 end); |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
98 |
11594
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
99 local buckets = {}; |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
100 for n = 10, 40, 2 do |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
101 local exp = math.floor(2 ^ n); |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
102 table.insert(buckets, exp); |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
103 if exp >= file_size_limit then break end |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
104 end |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
105 local measure_uploads = module:measure("upload", "sizes", {buckets = buckets}); |
11355
89efa3f2966b
mod_http_file_share: Collect statistics of files uploaded
Kim Alvefur <zash@zash.se>
parents:
11350
diff
changeset
|
106 |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
107 -- Convenience wrapper for logging file sizes |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
108 local function B(bytes) |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
109 if bytes ~= bytes then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
110 return "unknown" |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
111 elseif bytes == unlimited then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
112 return "unlimited"; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
113 end |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
114 return hi.format(bytes, "B", "b"); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
115 end |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
116 |
11325
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
117 local function get_filename(slot, create) |
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
118 return dm.getpath(slot, module.host, module.name, "bin", create) |
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
119 end |
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
120 |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
121 function get_daily_quota(uploader) |
11347
5b3ad3c7fe47
mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents:
11346
diff
changeset
|
122 local now = os.time(); |
5b3ad3c7fe47
mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents:
11346
diff
changeset
|
123 local max_age = now - 86400; |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
124 local cached = quota_cache:get(uploader); |
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
125 if cached and cached.time > max_age then |
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
126 return cached.size; |
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
127 end |
11347
5b3ad3c7fe47
mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents:
11346
diff
changeset
|
128 local iter, err = uploads:find(nil, {with = uploader; start = max_age }); |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
129 if not iter then return iter, err; end |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
130 local total_bytes = 0; |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
131 local oldest_upload = now; |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
132 for _, slot, when in iter do |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
133 local size = tonumber(slot.attr.size); |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
134 if size then total_bytes = total_bytes + size; end |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
135 if when < oldest_upload then oldest_upload = when; end |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
136 end |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
137 -- If there were no uploads then we end up caching [now, 0], which is fine |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
138 -- since we increase the size on new uploads |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
139 quota_cache:set(uploader, { time = oldest_upload, size = total_bytes }); |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
140 return total_bytes; |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
141 end |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
142 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
143 function may_upload(uploader, filename, filesize, filetype) -- > boolean, error |
11311
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
144 local uploader_host = jid.host(uploader); |
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
145 if not ((access:empty() and prosody.hosts[uploader_host]) or access:contains(uploader) or access:contains(uploader_host)) then |
11312
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
146 return false, upload_errors.new("access"); |
11311
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
147 end |
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
148 |
11313
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
149 if not filename or filename:find"/" then |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
150 -- On Linux, only '/' and '\0' are invalid in filenames and NUL can't be in XML |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
151 return false, upload_errors.new("filename"); |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
152 end |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
153 |
11319
a4b299e37909
mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents:
11318
diff
changeset
|
154 if not filesize or filesize < 0 or filesize % 1 ~= 0 then |
a4b299e37909
mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents:
11318
diff
changeset
|
155 return false, upload_errors.new("filesizefmt"); |
a4b299e37909
mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents:
11318
diff
changeset
|
156 end |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
157 if filesize > file_size_limit then |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
158 return false, upload_errors.new("filesize"); |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
159 end |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
160 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
161 if total_storage_usage + filesize > total_storage_limit then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
162 module:log("warn", "Global storage quota reached, at %s / %s!", B(total_storage_usage), B(total_storage_limit)); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
163 return false, upload_errors.new("outofdisk"); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
164 end |
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
165 |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
166 local uploader_quota = get_daily_quota(uploader); |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
167 if uploader_quota + filesize > daily_quota then |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
168 return false, upload_errors.new("quota"); |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
169 end |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
170 |
11315
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
171 if not ( file_types:empty() or file_types:contains(filetype) or file_types:contains(filetype:gsub("/.*", "/*")) ) then |
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
172 return false, upload_errors.new("filetype"); |
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
173 end |
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
174 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
175 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
176 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
177 |
11350
3287dbdde33e
mod_http_file_share: Reorder arguments
Kim Alvefur <zash@zash.se>
parents:
11349
diff
changeset
|
178 function get_authz(slot, uploader, filename, filesize, filetype) |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
179 return create_jwt({ |
11501
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
180 -- token properties |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
181 sub = uploader; |
11501
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
182 |
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
183 -- slot properties |
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
184 slot = slot; |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
185 expires = expiry >= 0 and (os.time()+expiry) or nil; |
11501
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
186 -- file properties |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
187 filename = filename; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
188 filesize = filesize; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
189 filetype = filetype; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
190 }); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
191 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
192 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
193 function get_url(slot, filename) |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
194 local base_url = external_base_url or module:http_url(); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
195 local slot_url = url.parse(base_url); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
196 slot_url.path = url.parse_path(slot_url.path or "/"); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
197 t_insert(slot_url.path, slot); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
198 if filename then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
199 t_insert(slot_url.path, filename); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
200 slot_url.path.is_directory = false; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
201 else |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
202 slot_url.path.is_directory = true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
203 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
204 slot_url.path = url.build_path(slot_url.path); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
205 return url.build(slot_url); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
206 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
207 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
208 function handle_slot_request(event) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
209 local stanza, origin = event.stanza, event.origin; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
210 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
211 local request = st.clone(stanza.tags[1], true); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
212 local filename = request.attr.filename; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
213 local filesize = tonumber(request.attr.size); |
11320
817cadf6be92
mod_http_file_share: Handle content-type being optional
Kim Alvefur <zash@zash.se>
parents:
11319
diff
changeset
|
214 local filetype = request.attr["content-type"] or "application/octet-stream"; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
215 local uploader = jid.bare(stanza.attr.from); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
216 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
217 local may, why_not = may_upload(uploader, filename, filesize, filetype); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
218 if not may then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
219 origin.send(st.error_reply(stanza, why_not)); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
220 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
221 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
222 |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
223 module:log("info", "Issuing upload slot to %s for %s", uploader, B(filesize)); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
224 local slot, storage_err = errors.coerce(uploads:append(nil, nil, request, os.time(), uploader)) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
225 if not slot then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
226 origin.send(st.error_reply(stanza, storage_err)); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
227 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
228 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
229 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
230 total_storage_usage = total_storage_usage + filesize; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
231 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
232 |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
233 local cached_quota = quota_cache:get(uploader); |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
234 if cached_quota and cached_quota.time > os.time()-86400 then |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
235 cached_quota.size = cached_quota.size + filesize; |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
236 quota_cache:set(uploader, cached_quota); |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
237 end |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
238 |
11350
3287dbdde33e
mod_http_file_share: Reorder arguments
Kim Alvefur <zash@zash.se>
parents:
11349
diff
changeset
|
239 local authz = get_authz(slot, uploader, filename, filesize, filetype); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
240 local slot_url = get_url(slot, filename); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
241 local upload_url = slot_url; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
242 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
243 local reply = st.reply(stanza) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
244 :tag("slot", { xmlns = namespace }) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
245 :tag("get", { url = slot_url }):up() |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
246 :tag("put", { url = upload_url }) |
11322
4ade9810ce35
mod_http_file_share: Move Authorization type string
Kim Alvefur <zash@zash.se>
parents:
11321
diff
changeset
|
247 :text_tag("header", "Bearer "..authz, {name="Authorization"}) |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
248 :reset(); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
249 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
250 origin.send(reply); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
251 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
252 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
253 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
254 function handle_upload(event, path) -- PUT /upload/:slot |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
255 local request = event.request; |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
256 local upload_info = request.http_file_share_upload_info; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
257 |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
258 if not upload_info then -- Initial handling of request |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
259 local authz = request.headers.authorization; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
260 if authz then |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
261 authz = authz:match("^Bearer (.*)") |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
262 end |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
263 if not authz then |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
264 module:log("debug", "Missing or malformed Authorization header"); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
265 event.response.headers.www_authenticate = "Bearer"; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
266 return 401; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
267 end |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
268 local authed, authed_upload_info = verify_jwt(authz); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
269 if not authed then |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
270 module:log("debug", "Unauthorized or invalid token: %s, %q", authz, authed_upload_info); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
271 return 401; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
272 end |
12722
cd993fd7b60d
mod_http_file_share: Use correct variable name (thanks riau.sni)
Matthew Wild <mwild1@gmail.com>
parents:
12708
diff
changeset
|
273 if not path or authed_upload_info.slot ~= path:match("^[^/]+") then |
cd993fd7b60d
mod_http_file_share: Use correct variable name (thanks riau.sni)
Matthew Wild <mwild1@gmail.com>
parents:
12708
diff
changeset
|
274 module:log("debug", "Invalid upload slot: %q, path: %q", authed_upload_info.slot, path); |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
275 return 400; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
276 end |
12722
cd993fd7b60d
mod_http_file_share: Use correct variable name (thanks riau.sni)
Matthew Wild <mwild1@gmail.com>
parents:
12708
diff
changeset
|
277 if request.headers.content_length and tonumber(request.headers.content_length) ~= authed_upload_info.filesize then |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
278 return 413; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
279 -- Note: We don't know the size if the upload is streamed in chunked encoding, |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
280 -- so we also check the final file size on completion. |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
281 end |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
282 upload_info = authed_upload_info; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
283 request.http_file_share_upload_info = upload_info; |
11323
a853a018eede
mod_http_file_share: Validate file size early in HTTP PUT request
Kim Alvefur <zash@zash.se>
parents:
11322
diff
changeset
|
284 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
285 |
11325
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
286 local filename = get_filename(upload_info.slot, true); |
11324
494761f5d7da
mod_http_file_share: Use '.bin' file extension
Kim Alvefur <zash@zash.se>
parents:
11323
diff
changeset
|
287 |
11375
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
288 do |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
289 -- check if upload has been completed already |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
290 -- we want to allow retry of a failed upload attempt, but not after it's been completed |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
291 local f = io.open(filename, "r"); |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
292 if f then |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
293 f:close(); |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
294 return 409; |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
295 end |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
296 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
297 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
298 if not request.body_sink then |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
299 module:log("debug", "Preparing to receive upload into %q, expecting %s", filename, B(upload_info.filesize)); |
11500
21706a581b8a
mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents:
11499
diff
changeset
|
300 local fh, err = io.open(filename.."~", "w"); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
301 if not fh then |
11500
21706a581b8a
mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents:
11499
diff
changeset
|
302 module:log("error", "Could not open file for writing: %s", err); |
21706a581b8a
mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents:
11499
diff
changeset
|
303 return 500; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
304 end |
11865
77bbbd4263d7
mod_http_file_share: Silence luacheck warning
Kim Alvefur <zash@zash.se>
parents:
11864
diff
changeset
|
305 function event.response:on_destroy() -- luacheck: ignore 212/self |
11864
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
306 -- Clean up incomplete upload |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
307 if io.type(fh) == "file" then -- still open |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
308 fh:close(); |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
309 os.remove(filename.."~"); |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
310 end |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
311 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
312 request.body_sink = fh; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
313 if request.body == false then |
11374
5b8aec0609f0
mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents:
11357
diff
changeset
|
314 if request.headers.expect == "100-continue" then |
5b8aec0609f0
mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents:
11357
diff
changeset
|
315 request.conn:write("HTTP/1.1 100 Continue\r\n\r\n"); |
5b8aec0609f0
mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents:
11357
diff
changeset
|
316 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
317 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
318 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
319 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
320 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
321 if request.body then |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
322 module:log("debug", "Complete upload available, %s", B(#request.body)); |
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
323 -- Small enough to have been uploaded already |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
324 local written, err = errors.coerce(request.body_sink:write(request.body)); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
325 if not written then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
326 return err; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
327 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
328 request.body = nil; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
329 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
330 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
331 if request.body_sink then |
11316
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
332 local final_size = request.body_sink:seek(); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
333 local uploaded, err = errors.coerce(request.body_sink:close()); |
11316
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
334 if final_size ~= upload_info.filesize then |
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
335 -- Could be too short as well, but we say the same thing |
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
336 uploaded, err = false, 413; |
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
337 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
338 if uploaded then |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
339 module:log("debug", "Upload of %q completed, %s", filename, B(final_size)); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
340 assert(os.rename(filename.."~", filename)); |
11355
89efa3f2966b
mod_http_file_share: Collect statistics of files uploaded
Kim Alvefur <zash@zash.se>
parents:
11350
diff
changeset
|
341 measure_uploads(final_size); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
342 |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
343 upload_cache:set(upload_info.slot, { |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
344 name = upload_info.filename; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
345 size = tostring(upload_info.filesize); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
346 type = upload_info.filetype; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
347 time = os.time(); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
348 }); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
349 return 201; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
350 else |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
351 assert(os.remove(filename.."~")); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
352 return err; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
353 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
354 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
355 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
356 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
357 |
11357
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
358 local download_cache_hit = module:measure("download_cache_hit", "rate"); |
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
359 local download_cache_miss = module:measure("download_cache_miss", "rate"); |
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
360 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
361 function handle_download(event, path) -- GET /uploads/:slot+filename |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
362 local request, response = event.request, event.response; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
363 local slot_id = path:match("^[^/]+"); |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
364 local basename, filetime, filetype, filesize; |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
365 local cached = upload_cache:get(slot_id); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
366 if cached then |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
367 module:log("debug", "Cache hit"); |
11357
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
368 download_cache_hit(); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
369 basename = cached.name; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
370 filesize = cached.size; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
371 filetype = cached.type; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
372 filetime = cached.time; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
373 upload_cache:set(slot_id, cached); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
374 -- TODO cache negative hits? |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
375 else |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
376 module:log("debug", "Cache miss"); |
11357
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
377 download_cache_miss(); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
378 local slot, when = errors.coerce(uploads:get(nil, slot_id)); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
379 if not slot then |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
380 module:log("debug", "uploads:get(%q) --> not-found, %s", slot_id, when); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
381 else |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
382 module:log("debug", "uploads:get(%q) --> %s, %d", slot_id, slot, when); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
383 basename = slot.attr.filename; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
384 filesize = slot.attr.size; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
385 filetype = slot.attr["content-type"]; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
386 filetime = when; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
387 upload_cache:set(slot_id, { |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
388 name = basename; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
389 size = slot.attr.size; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
390 type = filetype; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
391 time = when; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
392 }); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
393 end |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
394 end |
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
395 if not basename then |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
396 return 404; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
397 end |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
398 local last_modified = os.date('!%a, %d %b %Y %H:%M:%S GMT', filetime); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
399 if request.headers.if_modified_since == last_modified then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
400 return 304; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
401 end |
11325
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
402 local filename = get_filename(slot_id); |
11493
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
403 local handle, ferr = io.open(filename); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
404 if not handle then |
11499
a8cbbbb1f165
mod_http_file_share: Fix logging of error opening file
Kim Alvefur <zash@zash.se>
parents:
11496
diff
changeset
|
405 module:log("error", "Could not open file for reading: %s", ferr); |
11493
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
406 -- This can be because the upload slot wasn't used, or the file disappeared |
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
407 -- somehow, or permission issues. |
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
408 return 410; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
409 end |
11333
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
410 |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
411 local request_range = request.headers.range; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
412 local response_range; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
413 if request_range then |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
414 local range_start, range_end = request_range:match("^bytes=(%d+)%-(%d*)$") |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
415 -- Only support resumption, ie ranges from somewhere in the middle until the end of the file. |
11568
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
416 if (range_start and range_start ~= "0") and (range_end == "" or range_end == filesize) then |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
417 local pos, size = tonumber(range_start), tonumber(filesize); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
418 local new_pos = pos < size and handle:seek("set", pos); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
419 if new_pos and new_pos < size then |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
420 response_range = "bytes "..range_start.."-"..filesize.."/"..filesize; |
11568
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
421 filesize = string.format("%d", size-pos); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
422 else |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
423 handle:close(); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
424 return 416; |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
425 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
426 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
427 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
428 |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
429 |
11402
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
430 if not filetype then |
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
431 filetype = "application/octet-stream"; |
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
432 end |
11333
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
433 local disposition = "attachment"; |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
434 if safe_types:contains(filetype) or safe_types:contains(filetype:gsub("/.*", "/*")) then |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
435 disposition = "inline"; |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
436 end |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
437 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
438 response.headers.last_modified = last_modified; |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
439 response.headers.content_length = filesize; |
11402
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
440 response.headers.content_type = filetype; |
12227
88958c0ecab3
mod_http_file_share: Use alternate syntax for filename in Content-Disposition
Kim Alvefur <zash@zash.se>
parents:
12179
diff
changeset
|
441 response.headers.content_disposition = string.format("%s; filename*=UTF-8''%s", disposition, urlencode(basename)); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
442 |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
443 if response_range then |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
444 response.status_code = 206; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
445 response.headers.content_range = response_range; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
446 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
447 response.headers.accept_ranges = "bytes"; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
448 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
449 response.headers.cache_control = "max-age=31556952, immutable"; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
450 response.headers.content_security_policy = "default-src 'none'; frame-ancestors 'none';" |
11327
6f2b69469060
mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents:
11326
diff
changeset
|
451 response.headers.strict_transport_security = "max-age=31556952"; |
6f2b69469060
mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents:
11326
diff
changeset
|
452 response.headers.x_content_type_options = "nosniff"; |
11611
a6d1131ac833
mod_http_file_share: Update comment about x-frame-options
Kim Alvefur <zash@zash.se>
parents:
11594
diff
changeset
|
453 response.headers.x_frame_options = "DENY"; -- COMPAT IE missing support for CSP frame-ancestors |
11327
6f2b69469060
mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents:
11326
diff
changeset
|
454 response.headers.x_xss_protection = "1; mode=block"; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
455 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
456 return response:send_file(handle); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
457 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
458 |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
459 if expiry >= 0 and not external_base_url then |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
460 -- TODO HTTP DELETE to the external endpoint? |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
461 local array = require "prosody.util.array"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
462 local async = require "prosody.util.async"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
463 local ENOENT = require "prosody.util.pposix".ENOENT; |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
464 |
11329
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
465 local function sleep(t) |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
466 local wait, done = async.waiter(); |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
467 module:add_timer(t, done) |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
468 wait(); |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
469 end |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
470 |
11802
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
471 local prune_start = module:measure("prune", "times"); |
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
472 |
12008
c01532ae6a3b
mod_http_file_share: Fix to take retention time into account
Kim Alvefur <zash@zash.se>
parents:
12007
diff
changeset
|
473 module:daily("Remove expired files", function(_, current_time) |
11802
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
474 local prune_done = prune_start(); |
12008
c01532ae6a3b
mod_http_file_share: Fix to take retention time into account
Kim Alvefur <zash@zash.se>
parents:
12007
diff
changeset
|
475 local boundary_time = (current_time or os.time()) - expiry; |
12004
a2a0e00cbd24
mod_http_file_share: Back out 876e1b6d6ae4
Kim Alvefur <zash@zash.se>
parents:
12003
diff
changeset
|
476 local iter, total = assert(uploads:find(nil, {["end"] = boundary_time; total = true})); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
477 |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
478 if total == 0 then |
11343
f125ac529c22
mod_http_file_share: Clarify log message
Kim Alvefur <zash@zash.se>
parents:
11336
diff
changeset
|
479 module:log("info", "No expired uploaded files to prune"); |
11356
43e5429ab355
mod_http_file_share: Measure how long it takes to prune expired files
Kim Alvefur <zash@zash.se>
parents:
11355
diff
changeset
|
480 prune_done(); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
481 return; |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
482 end |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
483 |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
484 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time)); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
485 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
486 |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
487 local obsolete_uploads = array(); |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
488 local num_expired = 0; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
489 local size_sum = 0; |
11993
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
490 local problem_deleting = false; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
491 for slot_id, slot_info in iter do |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
492 num_expired = num_expired + 1; |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
493 upload_cache:set(slot_id, nil); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
494 local filename = get_filename(slot_id); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
495 local deleted, err, errno = os.remove(filename); |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
496 if deleted or errno == ENOENT then -- removed successfully or it was already gone |
11993
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
497 size_sum = size_sum + tonumber(slot_info.attr.size); |
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
498 obsolete_uploads:push(slot_id); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
499 else |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
500 module:log("error", "Could not prune expired file %q: %s", filename, err); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
501 problem_deleting = true; |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
502 end |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
503 if num_expired % 100 == 0 then sleep(0.1); end |
11993
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
504 end |
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
505 |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
506 -- obsolete_uploads now contains slot ids for which the files have been |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
507 -- removed and that needs to be cleared from the database |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
508 |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
509 local deletion_query = {["end"] = boundary_time}; |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
510 if not problem_deleting then |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
511 module:log("info", "All (%d, %s) expired files successfully pruned", num_expired, B(size_sum)); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
512 -- we can delete based on time |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
513 else |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
514 module:log("warn", "%d out of %d expired files could not be pruned", num_expired-#obsolete_uploads, num_expired); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
515 -- we'll need to delete only those entries where the files were |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
516 -- successfully removed, and then try again with the failed ones. |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
517 -- eventually the admin ought to notice and fix the permissions or |
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
518 -- whatever the problem is. |
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
519 deletion_query = {ids = obsolete_uploads}; |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
520 end |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
521 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
522 total_storage_usage = total_storage_usage - size_sum; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
523 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
524 persist_stats:set(nil, "total", total_storage_usage); |
11994
f9b2325f6b50
mod_http_file_share: Keep global storage use accurate longer.
Kim Alvefur <zash@zash.se>
parents:
11993
diff
changeset
|
525 |
11406
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
526 if #obsolete_uploads == 0 then |
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
527 module:log("debug", "No metadata to remove"); |
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
528 else |
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
529 local removed, err = uploads:delete(nil, deletion_query); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
530 |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
531 if removed == true or removed == num_expired or removed == #obsolete_uploads then |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
532 module:log("debug", "Expired upload metadata pruned successfully"); |
11406
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
533 else |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
534 module:log("error", "Problem removing metadata for expired files: %s", err); |
11406
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
535 end |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
536 end |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
537 |
11356
43e5429ab355
mod_http_file_share: Measure how long it takes to prune expired files
Kim Alvefur <zash@zash.se>
parents:
11355
diff
changeset
|
538 prune_done(); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
539 end); |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
540 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
541 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
542 local summary_start = module:measure("summary", "times"); |
11802
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
543 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
544 module:weekly("Calculate total storage usage", function() |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
545 local summary_done = summary_start(); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
546 local iter = assert(uploads:find(nil)); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
547 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
548 local count, sum = 0, 0; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
549 for _, file in iter do |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
550 sum = sum + tonumber(file.attr.size); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
551 count = count + 1; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
552 end |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
553 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
554 module:log("info", "Uploaded files total: %s in %d files", B(sum), count); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
555 if persist_stats:set(nil, "total", sum) then |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
556 total_storage_usage = sum; |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
557 else |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
558 total_storage_usage = unknown; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
559 end |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
560 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
561 summary_done(); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
562 end); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
563 |
11495
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
564 -- Reachable from the console |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
565 function check_files(query) |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
566 local issues = {}; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
567 local iter = assert(uploads:find(nil, query)); |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
568 for slot_id, file in iter do |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
569 local filename = get_filename(slot_id); |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
570 local size, err = lfs.attributes(filename, "size"); |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
571 if not size then |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
572 issues[filename] = err; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
573 elseif tonumber(file.attr.size) ~= size then |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
574 issues[filename] = "file size mismatch"; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
575 end |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
576 end |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
577 |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
578 return next(issues) == nil, issues; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
579 end |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
580 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
581 module:hook("iq-get/host/urn:xmpp:http:upload:0:request", handle_slot_request); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
582 |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
583 if not external_base_url then |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
584 module:provides("http", { |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
585 streaming_uploads = true; |
11398
a1f26d17d70f
mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents:
11394
diff
changeset
|
586 cors = { |
12444
b33558969b3e
mod_http (and dependent modules): Make CORS opt-in by default (fixes #1731)
Matthew Wild <mwild1@gmail.com>
parents:
12227
diff
changeset
|
587 enabled = true; |
11398
a1f26d17d70f
mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents:
11394
diff
changeset
|
588 credentials = true; |
11857
e080d6aa0b3b
mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents:
11853
diff
changeset
|
589 headers = { |
e080d6aa0b3b
mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents:
11853
diff
changeset
|
590 Authorization = true; |
e080d6aa0b3b
mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents:
11853
diff
changeset
|
591 }; |
11398
a1f26d17d70f
mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents:
11394
diff
changeset
|
592 }; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
593 route = { |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
594 ["PUT /*"] = handle_upload; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
595 ["GET /*"] = handle_download; |
11394
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
596 ["GET /"] = function (event) |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
597 return prosody.events.fire_event("http-message", { |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
598 response = event.response; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
599 --- |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
600 title = "Prosody HTTP Upload endpoint"; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
601 message = "This is where files will be uploaded to, and served from."; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
602 warning = not (event.request.secure) and "This endpoint is not considered secure!" or nil; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
603 }) or "This is the Prosody HTTP Upload endpoint."; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
604 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
605 } |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
606 }); |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
607 end |