Software /
code /
prosody
Annotate
plugins/mod_http_file_share.lua @ 13155:9a7523ea45cb
mod_storage_sql: Improve check for old table index on PostgreSQL
The "pg_indexes" view is much simpler to inspect than "pg_class"
author | Kim Alvefur <zash@zash.se> |
---|---|
date | Fri, 16 Jun 2023 23:57:15 +0200 |
parent | 13056:c38b1c63aa5c |
child | 13168:536055476912 |
rev | line source |
---|---|
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 -- Prosody IM |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 -- Copyright (C) 2021 Kim Alvefur |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
3 -- |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
4 -- This project is MIT/X11 licensed. Please see the |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 -- COPYING file in the source package for more information. |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
6 -- |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
7 -- XEP-0363: HTTP File Upload |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
8 -- Again, from the top! |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
9 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
10 local t_insert = table.insert; |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
11 local jid = require "prosody.util.jid"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
12 local st = require "prosody.util.stanza"; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
13 local url = require "socket.url"; |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
14 local dm = require "prosody.core.storagemanager".olddm; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
15 local errors = require "prosody.util.error"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
16 local dataform = require "prosody.util.dataforms".new; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
17 local urlencode = require "prosody.util.http".urlencode; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
18 local dt = require "prosody.util.datetime"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
19 local hi = require "prosody.util.human.units"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
20 local cache = require "prosody.util.cache"; |
11495
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
21 local lfs = require "lfs"; |
13056
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
22 local parse_duration = require "prosody.util.human.io".parse_duration; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
23 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
24 local unknown = math.abs(0/0); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
25 local unlimited = math.huge; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
26 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
27 local namespace = "urn:xmpp:http:upload:0"; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
28 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
29 module:depends("disco"); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
30 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
31 module:add_identity("store", "file", module:get_option_string("name", "HTTP File Upload")); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
32 module:add_feature(namespace); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
33 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
34 local uploads = module:open_store("uploads", "archive"); |
11999
9d2eab56f124
mod_http_file_share: Keep track of total storage use across restarts
Kim Alvefur <zash@zash.se>
parents:
11998
diff
changeset
|
35 local persist_stats = module:open_store("upload_stats", "map"); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
36 -- id, <request>, time, owner |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
37 |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
38 local secret = module:get_option_string(module.name.."_secret", require"prosody.util.id".long()); |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
39 local external_base_url = module:get_option_string(module.name .. "_base_url"); |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
40 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB |
11315
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
41 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {}); |
11333
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
42 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"}); |
13056
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
43 local expiry_str = module:get_option_string(module.name .. "_expires_after", "1w"); |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
44 local expiry, parse_err = parse_duration(expiry_str); |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
45 if expiry == nil then |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
46 module:log("error", "Could not parse "..module.name.."_expire_after string %q: %s", expiry_str, parse_err); |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
47 return false; |
c38b1c63aa5c
mod_http_file_share: use util.human.io.parse_duration
Jonas Schäfer <jonas@wielicki.name>
parents:
12977
diff
changeset
|
48 end |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
49 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
50 local total_storage_limit = module:get_option_number(module.name.."_global_quota", unlimited); |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
51 |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
52 local create_jwt, verify_jwt = require "prosody.util.jwt".init("HS256", secret); |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
53 |
11311
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
54 local access = module:get_option_set(module.name .. "_access", {}); |
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
55 |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
56 if not external_base_url then |
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
57 module:depends("http"); |
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
58 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
59 |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
60 module:add_extension(dataform { |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
61 { name = "FORM_TYPE", type = "hidden", value = namespace }, |
11873
2b85e4e7d389
mod_http_file_share: Move number coercion into util.dataforms
Kim Alvefur <zash@zash.se>
parents:
11865
diff
changeset
|
62 { name = "max-file-size", type = "text-single", datatype = "xs:integer" }, |
2b85e4e7d389
mod_http_file_share: Move number coercion into util.dataforms
Kim Alvefur <zash@zash.se>
parents:
11865
diff
changeset
|
63 }:form({ ["max-file-size"] = file_size_limit }, "result")); |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
64 |
11312
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
65 local upload_errors = errors.init(module.name, namespace, { |
11317
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
66 access = { type = "auth"; condition = "forbidden" }; |
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
67 filename = { type = "modify"; condition = "bad-request"; text = "Invalid filename" }; |
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
68 filetype = { type = "modify"; condition = "not-acceptable"; text = "File type not allowed" }; |
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
69 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large"; |
11318
3b16aba6285f
mod_http_file_share: Fix name of max-file-size tag
Kim Alvefur <zash@zash.se>
parents:
11317
diff
changeset
|
70 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) }; |
11330
f2c9492e3d25
mod_http_file_share: Fix the obligatory misplaced closing bracket (thanks scansion)
Kim Alvefur <zash@zash.se>
parents:
11329
diff
changeset
|
71 }; |
11345
0fec04b64a49
mod_http_file_share: Add missing semicolon
Kim Alvefur <zash@zash.se>
parents:
11343
diff
changeset
|
72 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; }; |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
73 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; }; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
74 outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" }; |
11312
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
75 }); |
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
76 |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
77 local upload_cache = cache.new(1024); |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
78 local quota_cache = cache.new(1024); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
79 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
80 local total_storage_usage = unknown; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
81 |
11491
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
82 local measure_upload_cache_size = module:measure("upload_cache", "amount"); |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
83 local measure_quota_cache_size = module:measure("quota_cache", "amount"); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
84 local measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" }); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
85 |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
86 do |
11999
9d2eab56f124
mod_http_file_share: Keep track of total storage use across restarts
Kim Alvefur <zash@zash.se>
parents:
11998
diff
changeset
|
87 local total, err = persist_stats:get(nil, "total"); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
88 if not err then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
89 total_storage_usage = tonumber(total) or 0; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
90 end |
11794
5d925f340ae6
mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents:
11784
diff
changeset
|
91 end |
11491
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
92 |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
93 module:hook_global("stats-update", function () |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
94 measure_upload_cache_size(upload_cache:count()); |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
95 measure_quota_cache_size(quota_cache:count()); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
96 measure_total_storage_usage(total_storage_usage); |
11491
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
97 end); |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
98 |
11594
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
99 local buckets = {}; |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
100 for n = 10, 40, 2 do |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
101 local exp = math.floor(2 ^ n); |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
102 table.insert(buckets, exp); |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
103 if exp >= file_size_limit then break end |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
104 end |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
105 local measure_uploads = module:measure("upload", "sizes", {buckets = buckets}); |
11355
89efa3f2966b
mod_http_file_share: Collect statistics of files uploaded
Kim Alvefur <zash@zash.se>
parents:
11350
diff
changeset
|
106 |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
107 -- Convenience wrapper for logging file sizes |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
108 local function B(bytes) |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
109 if bytes ~= bytes then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
110 return "unknown" |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
111 elseif bytes == unlimited then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
112 return "unlimited"; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
113 end |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
114 return hi.format(bytes, "B", "b"); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
115 end |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
116 |
11325
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
117 local function get_filename(slot, create) |
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
118 return dm.getpath(slot, module.host, module.name, "bin", create) |
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
119 end |
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
120 |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
121 function get_daily_quota(uploader) |
11347
5b3ad3c7fe47
mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents:
11346
diff
changeset
|
122 local now = os.time(); |
5b3ad3c7fe47
mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents:
11346
diff
changeset
|
123 local max_age = now - 86400; |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
124 local cached = quota_cache:get(uploader); |
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
125 if cached and cached.time > max_age then |
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
126 return cached.size; |
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
127 end |
11347
5b3ad3c7fe47
mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents:
11346
diff
changeset
|
128 local iter, err = uploads:find(nil, {with = uploader; start = max_age }); |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
129 if not iter then return iter, err; end |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
130 local total_bytes = 0; |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
131 local oldest_upload = now; |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
132 for _, slot, when in iter do |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
133 local size = tonumber(slot.attr.size); |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
134 if size then total_bytes = total_bytes + size; end |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
135 if when < oldest_upload then oldest_upload = when; end |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
136 end |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
137 -- If there were no uploads then we end up caching [now, 0], which is fine |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
138 -- since we increase the size on new uploads |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
139 quota_cache:set(uploader, { time = oldest_upload, size = total_bytes }); |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
140 return total_bytes; |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
141 end |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
142 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
143 function may_upload(uploader, filename, filesize, filetype) -- > boolean, error |
11311
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
144 local uploader_host = jid.host(uploader); |
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
145 if not ((access:empty() and prosody.hosts[uploader_host]) or access:contains(uploader) or access:contains(uploader_host)) then |
11312
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
146 return false, upload_errors.new("access"); |
11311
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
147 end |
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
148 |
11313
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
149 if not filename or filename:find"/" then |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
150 -- On Linux, only '/' and '\0' are invalid in filenames and NUL can't be in XML |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
151 return false, upload_errors.new("filename"); |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
152 end |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
153 |
11319
a4b299e37909
mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents:
11318
diff
changeset
|
154 if not filesize or filesize < 0 or filesize % 1 ~= 0 then |
a4b299e37909
mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents:
11318
diff
changeset
|
155 return false, upload_errors.new("filesizefmt"); |
a4b299e37909
mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents:
11318
diff
changeset
|
156 end |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
157 if filesize > file_size_limit then |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
158 return false, upload_errors.new("filesize"); |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
159 end |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
160 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
161 if total_storage_usage + filesize > total_storage_limit then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
162 module:log("warn", "Global storage quota reached, at %s / %s!", B(total_storage_usage), B(total_storage_limit)); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
163 return false, upload_errors.new("outofdisk"); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
164 end |
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
165 |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
166 local uploader_quota = get_daily_quota(uploader); |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
167 if uploader_quota + filesize > daily_quota then |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
168 return false, upload_errors.new("quota"); |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
169 end |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
170 |
11315
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
171 if not ( file_types:empty() or file_types:contains(filetype) or file_types:contains(filetype:gsub("/.*", "/*")) ) then |
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
172 return false, upload_errors.new("filetype"); |
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
173 end |
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
174 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
175 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
176 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
177 |
11350
3287dbdde33e
mod_http_file_share: Reorder arguments
Kim Alvefur <zash@zash.se>
parents:
11349
diff
changeset
|
178 function get_authz(slot, uploader, filename, filesize, filetype) |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
179 return create_jwt({ |
11501
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
180 -- token properties |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
181 sub = uploader; |
11501
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
182 |
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
183 -- slot properties |
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
184 slot = slot; |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
185 expires = expiry >= 0 and (os.time()+expiry) or nil; |
11501
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
186 -- file properties |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
187 filename = filename; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
188 filesize = filesize; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
189 filetype = filetype; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
190 }); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
191 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
192 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
193 function get_url(slot, filename) |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
194 local base_url = external_base_url or module:http_url(); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
195 local slot_url = url.parse(base_url); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
196 slot_url.path = url.parse_path(slot_url.path or "/"); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
197 t_insert(slot_url.path, slot); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
198 if filename then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
199 t_insert(slot_url.path, filename); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
200 slot_url.path.is_directory = false; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
201 else |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
202 slot_url.path.is_directory = true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
203 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
204 slot_url.path = url.build_path(slot_url.path); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
205 return url.build(slot_url); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
206 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
207 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
208 function handle_slot_request(event) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
209 local stanza, origin = event.stanza, event.origin; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
210 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
211 local request = st.clone(stanza.tags[1], true); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
212 local filename = request.attr.filename; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
213 local filesize = tonumber(request.attr.size); |
11320
817cadf6be92
mod_http_file_share: Handle content-type being optional
Kim Alvefur <zash@zash.se>
parents:
11319
diff
changeset
|
214 local filetype = request.attr["content-type"] or "application/octet-stream"; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
215 local uploader = jid.bare(stanza.attr.from); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
216 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
217 local may, why_not = may_upload(uploader, filename, filesize, filetype); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
218 if not may then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
219 origin.send(st.error_reply(stanza, why_not)); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
220 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
221 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
222 |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
223 module:log("info", "Issuing upload slot to %s for %s", uploader, B(filesize)); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
224 local slot, storage_err = errors.coerce(uploads:append(nil, nil, request, os.time(), uploader)) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
225 if not slot then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
226 origin.send(st.error_reply(stanza, storage_err)); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
227 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
228 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
229 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
230 total_storage_usage = total_storage_usage + filesize; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
231 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
232 |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
233 local cached_quota = quota_cache:get(uploader); |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
234 if cached_quota and cached_quota.time > os.time()-86400 then |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
235 cached_quota.size = cached_quota.size + filesize; |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
236 quota_cache:set(uploader, cached_quota); |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
237 end |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
238 |
11350
3287dbdde33e
mod_http_file_share: Reorder arguments
Kim Alvefur <zash@zash.se>
parents:
11349
diff
changeset
|
239 local authz = get_authz(slot, uploader, filename, filesize, filetype); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
240 local slot_url = get_url(slot, filename); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
241 local upload_url = slot_url; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
242 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
243 local reply = st.reply(stanza) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
244 :tag("slot", { xmlns = namespace }) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
245 :tag("get", { url = slot_url }):up() |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
246 :tag("put", { url = upload_url }) |
11322
4ade9810ce35
mod_http_file_share: Move Authorization type string
Kim Alvefur <zash@zash.se>
parents:
11321
diff
changeset
|
247 :text_tag("header", "Bearer "..authz, {name="Authorization"}) |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
248 :reset(); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
249 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
250 origin.send(reply); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
251 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
252 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
253 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
254 function handle_upload(event, path) -- PUT /upload/:slot |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
255 local request = event.request; |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
256 local upload_info = request.http_file_share_upload_info; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
257 |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
258 if not upload_info then -- Initial handling of request |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
259 local authz = request.headers.authorization; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
260 if authz then |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
261 authz = authz:match("^Bearer (.*)") |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
262 end |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
263 if not authz then |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
264 module:log("debug", "Missing or malformed Authorization header"); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
265 event.response.headers.www_authenticate = "Bearer"; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
266 return 401; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
267 end |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
268 local authed, authed_upload_info = verify_jwt(authz); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
269 if not authed then |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
270 module:log("debug", "Unauthorized or invalid token: %s, %q", authz, authed_upload_info); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
271 return 401; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
272 end |
12722
cd993fd7b60d
mod_http_file_share: Use correct variable name (thanks riau.sni)
Matthew Wild <mwild1@gmail.com>
parents:
12708
diff
changeset
|
273 if not path or authed_upload_info.slot ~= path:match("^[^/]+") then |
cd993fd7b60d
mod_http_file_share: Use correct variable name (thanks riau.sni)
Matthew Wild <mwild1@gmail.com>
parents:
12708
diff
changeset
|
274 module:log("debug", "Invalid upload slot: %q, path: %q", authed_upload_info.slot, path); |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
275 return 400; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
276 end |
12722
cd993fd7b60d
mod_http_file_share: Use correct variable name (thanks riau.sni)
Matthew Wild <mwild1@gmail.com>
parents:
12708
diff
changeset
|
277 if request.headers.content_length and tonumber(request.headers.content_length) ~= authed_upload_info.filesize then |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
278 return 413; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
279 -- Note: We don't know the size if the upload is streamed in chunked encoding, |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
280 -- so we also check the final file size on completion. |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
281 end |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
282 upload_info = authed_upload_info; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
283 request.http_file_share_upload_info = upload_info; |
11323
a853a018eede
mod_http_file_share: Validate file size early in HTTP PUT request
Kim Alvefur <zash@zash.se>
parents:
11322
diff
changeset
|
284 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
285 |
11325
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
286 local filename = get_filename(upload_info.slot, true); |
11324
494761f5d7da
mod_http_file_share: Use '.bin' file extension
Kim Alvefur <zash@zash.se>
parents:
11323
diff
changeset
|
287 |
11375
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
288 do |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
289 -- check if upload has been completed already |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
290 -- we want to allow retry of a failed upload attempt, but not after it's been completed |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
291 local f = io.open(filename, "r"); |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
292 if f then |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
293 f:close(); |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
294 return 409; |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
295 end |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
296 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
297 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
298 if not request.body_sink then |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
299 module:log("debug", "Preparing to receive upload into %q, expecting %s", filename, B(upload_info.filesize)); |
11500
21706a581b8a
mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents:
11499
diff
changeset
|
300 local fh, err = io.open(filename.."~", "w"); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
301 if not fh then |
11500
21706a581b8a
mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents:
11499
diff
changeset
|
302 module:log("error", "Could not open file for writing: %s", err); |
21706a581b8a
mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents:
11499
diff
changeset
|
303 return 500; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
304 end |
11865
77bbbd4263d7
mod_http_file_share: Silence luacheck warning
Kim Alvefur <zash@zash.se>
parents:
11864
diff
changeset
|
305 function event.response:on_destroy() -- luacheck: ignore 212/self |
11864
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
306 -- Clean up incomplete upload |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
307 if io.type(fh) == "file" then -- still open |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
308 fh:close(); |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
309 os.remove(filename.."~"); |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
310 end |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
311 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
312 request.body_sink = fh; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
313 if request.body == false then |
11374
5b8aec0609f0
mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents:
11357
diff
changeset
|
314 if request.headers.expect == "100-continue" then |
5b8aec0609f0
mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents:
11357
diff
changeset
|
315 request.conn:write("HTTP/1.1 100 Continue\r\n\r\n"); |
5b8aec0609f0
mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents:
11357
diff
changeset
|
316 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
317 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
318 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
319 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
320 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
321 if request.body then |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
322 module:log("debug", "Complete upload available, %s", B(#request.body)); |
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
323 -- Small enough to have been uploaded already |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
324 local written, err = errors.coerce(request.body_sink:write(request.body)); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
325 if not written then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
326 return err; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
327 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
328 request.body = nil; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
329 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
330 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
331 if request.body_sink then |
11316
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
332 local final_size = request.body_sink:seek(); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
333 local uploaded, err = errors.coerce(request.body_sink:close()); |
11316
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
334 if final_size ~= upload_info.filesize then |
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
335 -- Could be too short as well, but we say the same thing |
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
336 uploaded, err = false, 413; |
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
337 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
338 if uploaded then |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
339 module:log("debug", "Upload of %q completed, %s", filename, B(final_size)); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
340 assert(os.rename(filename.."~", filename)); |
11355
89efa3f2966b
mod_http_file_share: Collect statistics of files uploaded
Kim Alvefur <zash@zash.se>
parents:
11350
diff
changeset
|
341 measure_uploads(final_size); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
342 |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
343 upload_cache:set(upload_info.slot, { |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
344 name = upload_info.filename; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
345 size = tostring(upload_info.filesize); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
346 type = upload_info.filetype; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
347 time = os.time(); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
348 }); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
349 return 201; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
350 else |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
351 assert(os.remove(filename.."~")); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
352 return err; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
353 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
354 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
355 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
356 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
357 |
11357
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
358 local download_cache_hit = module:measure("download_cache_hit", "rate"); |
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
359 local download_cache_miss = module:measure("download_cache_miss", "rate"); |
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
360 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
361 function handle_download(event, path) -- GET /uploads/:slot+filename |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
362 local request, response = event.request, event.response; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
363 local slot_id = path:match("^[^/]+"); |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
364 local basename, filetime, filetype, filesize; |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
365 local cached = upload_cache:get(slot_id); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
366 if cached then |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
367 module:log("debug", "Cache hit"); |
11357
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
368 download_cache_hit(); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
369 basename = cached.name; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
370 filesize = cached.size; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
371 filetype = cached.type; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
372 filetime = cached.time; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
373 upload_cache:set(slot_id, cached); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
374 -- TODO cache negative hits? |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
375 else |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
376 module:log("debug", "Cache miss"); |
11357
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
377 download_cache_miss(); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
378 local slot, when = errors.coerce(uploads:get(nil, slot_id)); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
379 if not slot then |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
380 module:log("debug", "uploads:get(%q) --> not-found, %s", slot_id, when); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
381 else |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
382 module:log("debug", "uploads:get(%q) --> %s, %d", slot_id, slot, when); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
383 basename = slot.attr.filename; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
384 filesize = slot.attr.size; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
385 filetype = slot.attr["content-type"]; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
386 filetime = when; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
387 upload_cache:set(slot_id, { |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
388 name = basename; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
389 size = slot.attr.size; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
390 type = filetype; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
391 time = when; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
392 }); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
393 end |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
394 end |
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
395 if not basename then |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
396 return 404; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
397 end |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
398 local last_modified = os.date('!%a, %d %b %Y %H:%M:%S GMT', filetime); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
399 if request.headers.if_modified_since == last_modified then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
400 return 304; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
401 end |
11325
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
402 local filename = get_filename(slot_id); |
11493
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
403 local handle, ferr = io.open(filename); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
404 if not handle then |
11499
a8cbbbb1f165
mod_http_file_share: Fix logging of error opening file
Kim Alvefur <zash@zash.se>
parents:
11496
diff
changeset
|
405 module:log("error", "Could not open file for reading: %s", ferr); |
11493
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
406 -- This can be because the upload slot wasn't used, or the file disappeared |
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
407 -- somehow, or permission issues. |
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
408 return 410; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
409 end |
11333
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
410 |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
411 local request_range = request.headers.range; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
412 local response_range; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
413 if request_range then |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
414 local range_start, range_end = request_range:match("^bytes=(%d+)%-(%d*)$") |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
415 -- Only support resumption, ie ranges from somewhere in the middle until the end of the file. |
11568
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
416 if (range_start and range_start ~= "0") and (range_end == "" or range_end == filesize) then |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
417 local pos, size = tonumber(range_start), tonumber(filesize); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
418 local new_pos = pos < size and handle:seek("set", pos); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
419 if new_pos and new_pos < size then |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
420 response_range = "bytes "..range_start.."-"..filesize.."/"..filesize; |
11568
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
421 filesize = string.format("%d", size-pos); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
422 else |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
423 handle:close(); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
424 return 416; |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
425 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
426 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
427 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
428 |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
429 |
11402
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
430 if not filetype then |
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
431 filetype = "application/octet-stream"; |
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
432 end |
11333
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
433 local disposition = "attachment"; |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
434 if safe_types:contains(filetype) or safe_types:contains(filetype:gsub("/.*", "/*")) then |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
435 disposition = "inline"; |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
436 end |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
437 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
438 response.headers.last_modified = last_modified; |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
439 response.headers.content_length = filesize; |
11402
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
440 response.headers.content_type = filetype; |
12227
88958c0ecab3
mod_http_file_share: Use alternate syntax for filename in Content-Disposition
Kim Alvefur <zash@zash.se>
parents:
12179
diff
changeset
|
441 response.headers.content_disposition = string.format("%s; filename*=UTF-8''%s", disposition, urlencode(basename)); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
442 |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
443 if response_range then |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
444 response.status_code = 206; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
445 response.headers.content_range = response_range; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
446 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
447 response.headers.accept_ranges = "bytes"; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
448 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
449 response.headers.cache_control = "max-age=31556952, immutable"; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
450 response.headers.content_security_policy = "default-src 'none'; frame-ancestors 'none';" |
11327
6f2b69469060
mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents:
11326
diff
changeset
|
451 response.headers.strict_transport_security = "max-age=31556952"; |
6f2b69469060
mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents:
11326
diff
changeset
|
452 response.headers.x_content_type_options = "nosniff"; |
11611
a6d1131ac833
mod_http_file_share: Update comment about x-frame-options
Kim Alvefur <zash@zash.se>
parents:
11594
diff
changeset
|
453 response.headers.x_frame_options = "DENY"; -- COMPAT IE missing support for CSP frame-ancestors |
11327
6f2b69469060
mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents:
11326
diff
changeset
|
454 response.headers.x_xss_protection = "1; mode=block"; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
455 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
456 return response:send_file(handle); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
457 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
458 |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
459 if expiry >= 0 and not external_base_url then |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
460 -- TODO HTTP DELETE to the external endpoint? |
12977
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
461 local array = require "prosody.util.array"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
462 local async = require "prosody.util.async"; |
74b9e05af71e
plugins: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents:
12722
diff
changeset
|
463 local ENOENT = require "prosody.util.pposix".ENOENT; |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
464 |
11329
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
465 local function sleep(t) |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
466 local wait, done = async.waiter(); |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
467 module:add_timer(t, done) |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
468 wait(); |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
469 end |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
470 |
11802
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
471 local prune_start = module:measure("prune", "times"); |
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
472 |
12008
c01532ae6a3b
mod_http_file_share: Fix to take retention time into account
Kim Alvefur <zash@zash.se>
parents:
12007
diff
changeset
|
473 module:daily("Remove expired files", function(_, current_time) |
11802
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
474 local prune_done = prune_start(); |
12008
c01532ae6a3b
mod_http_file_share: Fix to take retention time into account
Kim Alvefur <zash@zash.se>
parents:
12007
diff
changeset
|
475 local boundary_time = (current_time or os.time()) - expiry; |
12004
a2a0e00cbd24
mod_http_file_share: Back out 876e1b6d6ae4
Kim Alvefur <zash@zash.se>
parents:
12003
diff
changeset
|
476 local iter, total = assert(uploads:find(nil, {["end"] = boundary_time; total = true})); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
477 |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
478 if total == 0 then |
11343
f125ac529c22
mod_http_file_share: Clarify log message
Kim Alvefur <zash@zash.se>
parents:
11336
diff
changeset
|
479 module:log("info", "No expired uploaded files to prune"); |
11356
43e5429ab355
mod_http_file_share: Measure how long it takes to prune expired files
Kim Alvefur <zash@zash.se>
parents:
11355
diff
changeset
|
480 prune_done(); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
481 return; |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
482 end |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
483 |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
484 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time)); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
485 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
486 |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
487 local obsolete_uploads = array(); |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
488 local num_expired = 0; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
489 local size_sum = 0; |
11993
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
490 local problem_deleting = false; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
491 for slot_id, slot_info in iter do |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
492 num_expired = num_expired + 1; |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
493 upload_cache:set(slot_id, nil); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
494 local filename = get_filename(slot_id); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
495 local deleted, err, errno = os.remove(filename); |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
496 if deleted or errno == ENOENT then -- removed successfully or it was already gone |
11993
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
497 size_sum = size_sum + tonumber(slot_info.attr.size); |
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
498 obsolete_uploads:push(slot_id); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
499 else |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
500 module:log("error", "Could not prune expired file %q: %s", filename, err); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
501 problem_deleting = true; |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
502 end |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
503 if num_expired % 100 == 0 then sleep(0.1); end |
11993
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
504 end |
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
505 |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
506 -- obsolete_uploads now contains slot ids for which the files have been |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
507 -- removed and that needs to be cleared from the database |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
508 |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
509 local deletion_query = {["end"] = boundary_time}; |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
510 if not problem_deleting then |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
511 module:log("info", "All (%d, %s) expired files successfully pruned", num_expired, B(size_sum)); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
512 -- we can delete based on time |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
513 else |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
514 module:log("warn", "%d out of %d expired files could not be pruned", num_expired-#obsolete_uploads, num_expired); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
515 -- we'll need to delete only those entries where the files were |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
516 -- successfully removed, and then try again with the failed ones. |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
517 -- eventually the admin ought to notice and fix the permissions or |
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
518 -- whatever the problem is. |
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
519 deletion_query = {ids = obsolete_uploads}; |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
520 end |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
521 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
522 total_storage_usage = total_storage_usage - size_sum; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
523 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
524 persist_stats:set(nil, "total", total_storage_usage); |
11994
f9b2325f6b50
mod_http_file_share: Keep global storage use accurate longer.
Kim Alvefur <zash@zash.se>
parents:
11993
diff
changeset
|
525 |
11406
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
526 if #obsolete_uploads == 0 then |
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
527 module:log("debug", "No metadata to remove"); |
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
528 else |
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
529 local removed, err = uploads:delete(nil, deletion_query); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
530 |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
531 if removed == true or removed == num_expired or removed == #obsolete_uploads then |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
532 module:log("debug", "Expired upload metadata pruned successfully"); |
11406
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
533 else |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
534 module:log("error", "Problem removing metadata for expired files: %s", err); |
11406
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
535 end |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
536 end |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
537 |
11356
43e5429ab355
mod_http_file_share: Measure how long it takes to prune expired files
Kim Alvefur <zash@zash.se>
parents:
11355
diff
changeset
|
538 prune_done(); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
539 end); |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
540 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
541 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
542 local summary_start = module:measure("summary", "times"); |
11802
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
543 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
544 module:weekly("Calculate total storage usage", function() |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
545 local summary_done = summary_start(); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
546 local iter = assert(uploads:find(nil)); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
547 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
548 local count, sum = 0, 0; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
549 for _, file in iter do |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
550 sum = sum + tonumber(file.attr.size); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
551 count = count + 1; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
552 end |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
553 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
554 module:log("info", "Uploaded files total: %s in %d files", B(sum), count); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
555 if persist_stats:set(nil, "total", sum) then |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
556 total_storage_usage = sum; |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
557 else |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
558 total_storage_usage = unknown; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
559 end |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
560 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
561 summary_done(); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
562 end); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
563 |
11495
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
564 -- Reachable from the console |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
565 function check_files(query) |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
566 local issues = {}; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
567 local iter = assert(uploads:find(nil, query)); |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
568 for slot_id, file in iter do |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
569 local filename = get_filename(slot_id); |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
570 local size, err = lfs.attributes(filename, "size"); |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
571 if not size then |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
572 issues[filename] = err; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
573 elseif tonumber(file.attr.size) ~= size then |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
574 issues[filename] = "file size mismatch"; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
575 end |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
576 end |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
577 |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
578 return next(issues) == nil, issues; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
579 end |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
580 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
581 module:hook("iq-get/host/urn:xmpp:http:upload:0:request", handle_slot_request); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
582 |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
583 if not external_base_url then |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
584 module:provides("http", { |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
585 streaming_uploads = true; |
11398
a1f26d17d70f
mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents:
11394
diff
changeset
|
586 cors = { |
12444
b33558969b3e
mod_http (and dependent modules): Make CORS opt-in by default (fixes #1731)
Matthew Wild <mwild1@gmail.com>
parents:
12227
diff
changeset
|
587 enabled = true; |
11398
a1f26d17d70f
mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents:
11394
diff
changeset
|
588 credentials = true; |
11857
e080d6aa0b3b
mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents:
11853
diff
changeset
|
589 headers = { |
e080d6aa0b3b
mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents:
11853
diff
changeset
|
590 Authorization = true; |
e080d6aa0b3b
mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents:
11853
diff
changeset
|
591 }; |
11398
a1f26d17d70f
mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents:
11394
diff
changeset
|
592 }; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
593 route = { |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
594 ["PUT /*"] = handle_upload; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
595 ["GET /*"] = handle_download; |
11394
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
596 ["GET /"] = function (event) |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
597 return prosody.events.fire_event("http-message", { |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
598 response = event.response; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
599 --- |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
600 title = "Prosody HTTP Upload endpoint"; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
601 message = "This is where files will be uploaded to, and served from."; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
602 warning = not (event.request.secure) and "This endpoint is not considered secure!" or nil; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
603 }) or "This is the Prosody HTTP Upload endpoint."; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
604 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
605 } |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
606 }); |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
607 end |