Software /
code /
prosody
Annotate
plugins/mod_http_file_share.lua @ 12956:52fcdfe710ca
storagemanager: Add keyval+ (combined keyval + map) store type
This combines the two most common store types, which modules often end up
opening with both interfaces separately anyway.
As well as combining them, I've taken the opportunity to improve some of the
method names to make them clearer.
author | Matthew Wild <mwild1@gmail.com> |
---|---|
date | Tue, 27 Sep 2022 17:46:27 +0100 |
parent | 12722:cd993fd7b60d |
child | 12977:74b9e05af71e |
rev | line source |
---|---|
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 -- Prosody IM |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 -- Copyright (C) 2021 Kim Alvefur |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
3 -- |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
4 -- This project is MIT/X11 licensed. Please see the |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 -- COPYING file in the source package for more information. |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
6 -- |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
7 -- XEP-0363: HTTP File Upload |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
8 -- Again, from the top! |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
9 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
10 local t_insert = table.insert; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
11 local jid = require "util.jid"; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
12 local st = require "util.stanza"; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
13 local url = require "socket.url"; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
14 local dm = require "core.storagemanager".olddm; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
15 local errors = require "util.error"; |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
16 local dataform = require "util.dataforms".new; |
12227
88958c0ecab3
mod_http_file_share: Use alternate syntax for filename in Content-Disposition
Kim Alvefur <zash@zash.se>
parents:
12179
diff
changeset
|
17 local urlencode = require "util.http".urlencode; |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
18 local dt = require "util.datetime"; |
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
19 local hi = require "util.human.units"; |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
20 local cache = require "util.cache"; |
11495
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
21 local lfs = require "lfs"; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
22 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
23 local unknown = math.abs(0/0); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
24 local unlimited = math.huge; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
25 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
26 local namespace = "urn:xmpp:http:upload:0"; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
27 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
28 module:depends("disco"); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
29 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
30 module:add_identity("store", "file", module:get_option_string("name", "HTTP File Upload")); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
31 module:add_feature(namespace); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
32 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
33 local uploads = module:open_store("uploads", "archive"); |
11999
9d2eab56f124
mod_http_file_share: Keep track of total storage use across restarts
Kim Alvefur <zash@zash.se>
parents:
11998
diff
changeset
|
34 local persist_stats = module:open_store("upload_stats", "map"); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
35 -- id, <request>, time, owner |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
36 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
37 local secret = module:get_option_string(module.name.."_secret", require"util.id".long()); |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
38 local external_base_url = module:get_option_string(module.name .. "_base_url"); |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
39 local file_size_limit = module:get_option_number(module.name .. "_size_limit", 10 * 1024 * 1024); -- 10 MB |
11315
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
40 local file_types = module:get_option_set(module.name .. "_allowed_file_types", {}); |
11333
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
41 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"}); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
42 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400); |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
43 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
44 local total_storage_limit = module:get_option_number(module.name.."_global_quota", unlimited); |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
45 |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
46 local create_jwt, verify_jwt = require "util.jwt".init("HS256", secret); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
47 |
11311
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
48 local access = module:get_option_set(module.name .. "_access", {}); |
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
49 |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
50 if not external_base_url then |
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
51 module:depends("http"); |
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
52 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
53 |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
54 module:add_extension(dataform { |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
55 { name = "FORM_TYPE", type = "hidden", value = namespace }, |
11873
2b85e4e7d389
mod_http_file_share: Move number coercion into util.dataforms
Kim Alvefur <zash@zash.se>
parents:
11865
diff
changeset
|
56 { name = "max-file-size", type = "text-single", datatype = "xs:integer" }, |
2b85e4e7d389
mod_http_file_share: Move number coercion into util.dataforms
Kim Alvefur <zash@zash.se>
parents:
11865
diff
changeset
|
57 }:form({ ["max-file-size"] = file_size_limit }, "result")); |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
58 |
11312
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
59 local upload_errors = errors.init(module.name, namespace, { |
11317
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
60 access = { type = "auth"; condition = "forbidden" }; |
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
61 filename = { type = "modify"; condition = "bad-request"; text = "Invalid filename" }; |
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
62 filetype = { type = "modify"; condition = "not-acceptable"; text = "File type not allowed" }; |
79e1f407b6f5
mod_http_file_share: Expand registry to fix extra tag
Kim Alvefur <zash@zash.se>
parents:
11316
diff
changeset
|
63 filesize = { type = "modify"; condition = "not-acceptable"; text = "File too large"; |
11318
3b16aba6285f
mod_http_file_share: Fix name of max-file-size tag
Kim Alvefur <zash@zash.se>
parents:
11317
diff
changeset
|
64 extra = {tag = st.stanza("file-too-large", {xmlns = namespace}):tag("max-file-size"):text(tostring(file_size_limit)) }; |
11330
f2c9492e3d25
mod_http_file_share: Fix the obligatory misplaced closing bracket (thanks scansion)
Kim Alvefur <zash@zash.se>
parents:
11329
diff
changeset
|
65 }; |
11345
0fec04b64a49
mod_http_file_share: Add missing semicolon
Kim Alvefur <zash@zash.se>
parents:
11343
diff
changeset
|
66 filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; }; |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
67 quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; }; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
68 outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" }; |
11312
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
69 }); |
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
70 |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
71 local upload_cache = cache.new(1024); |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
72 local quota_cache = cache.new(1024); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
73 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
74 local total_storage_usage = unknown; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
75 |
11491
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
76 local measure_upload_cache_size = module:measure("upload_cache", "amount"); |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
77 local measure_quota_cache_size = module:measure("quota_cache", "amount"); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
78 local measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" }); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
79 |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
80 do |
11999
9d2eab56f124
mod_http_file_share: Keep track of total storage use across restarts
Kim Alvefur <zash@zash.se>
parents:
11998
diff
changeset
|
81 local total, err = persist_stats:get(nil, "total"); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
82 if not err then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
83 total_storage_usage = tonumber(total) or 0; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
84 end |
11794
5d925f340ae6
mod_http_file_share: Measure current total usage
Kim Alvefur <zash@zash.se>
parents:
11784
diff
changeset
|
85 end |
11491
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
86 |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
87 module:hook_global("stats-update", function () |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
88 measure_upload_cache_size(upload_cache:count()); |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
89 measure_quota_cache_size(quota_cache:count()); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
90 measure_total_storage_usage(total_storage_usage); |
11491
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
91 end); |
c3fb802f9e45
mod_http_file_share: Report number of items in caches to statsmanager
Kim Alvefur <zash@zash.se>
parents:
11406
diff
changeset
|
92 |
11594
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
93 local buckets = {}; |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
94 for n = 10, 40, 2 do |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
95 local exp = math.floor(2 ^ n); |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
96 table.insert(buckets, exp); |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
97 if exp >= file_size_limit then break end |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
98 end |
19aac4247b03
mod_http_file_share: Build list of measuring buckets for configured size limit
Kim Alvefur <zash@zash.se>
parents:
11568
diff
changeset
|
99 local measure_uploads = module:measure("upload", "sizes", {buckets = buckets}); |
11355
89efa3f2966b
mod_http_file_share: Collect statistics of files uploaded
Kim Alvefur <zash@zash.se>
parents:
11350
diff
changeset
|
100 |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
101 -- Convenience wrapper for logging file sizes |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
102 local function B(bytes) |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
103 if bytes ~= bytes then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
104 return "unknown" |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
105 elseif bytes == unlimited then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
106 return "unlimited"; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
107 end |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
108 return hi.format(bytes, "B", "b"); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
109 end |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
110 |
11325
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
111 local function get_filename(slot, create) |
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
112 return dm.getpath(slot, module.host, module.name, "bin", create) |
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
113 end |
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
114 |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
115 function get_daily_quota(uploader) |
11347
5b3ad3c7fe47
mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents:
11346
diff
changeset
|
116 local now = os.time(); |
5b3ad3c7fe47
mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents:
11346
diff
changeset
|
117 local max_age = now - 86400; |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
118 local cached = quota_cache:get(uploader); |
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
119 if cached and cached.time > max_age then |
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
120 return cached.size; |
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
121 end |
11347
5b3ad3c7fe47
mod_http_file_share: Split out some variables for later reuse
Kim Alvefur <zash@zash.se>
parents:
11346
diff
changeset
|
122 local iter, err = uploads:find(nil, {with = uploader; start = max_age }); |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
123 if not iter then return iter, err; end |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
124 local total_bytes = 0; |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
125 local oldest_upload = now; |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
126 for _, slot, when in iter do |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
127 local size = tonumber(slot.attr.size); |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
128 if size then total_bytes = total_bytes + size; end |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
129 if when < oldest_upload then oldest_upload = when; end |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
130 end |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
131 -- If there were no uploads then we end up caching [now, 0], which is fine |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
132 -- since we increase the size on new uploads |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
133 quota_cache:set(uploader, { time = oldest_upload, size = total_bytes }); |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
134 return total_bytes; |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
135 end |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
136 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
137 function may_upload(uploader, filename, filesize, filetype) -- > boolean, error |
11311
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
138 local uploader_host = jid.host(uploader); |
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
139 if not ((access:empty() and prosody.hosts[uploader_host]) or access:contains(uploader) or access:contains(uploader_host)) then |
11312
aade4a6179a3
mod_http_file_share: Return proper error if unauthorized
Kim Alvefur <zash@zash.se>
parents:
11311
diff
changeset
|
140 return false, upload_errors.new("access"); |
11311
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
141 end |
9edda2026e57
mod_http_file_share: Add basic access control
Kim Alvefur <zash@zash.se>
parents:
11310
diff
changeset
|
142 |
11313
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
143 if not filename or filename:find"/" then |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
144 -- On Linux, only '/' and '\0' are invalid in filenames and NUL can't be in XML |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
145 return false, upload_errors.new("filename"); |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
146 end |
e53894d26092
mod_http_file_share: Validate that filename does not contain '/'
Kim Alvefur <zash@zash.se>
parents:
11312
diff
changeset
|
147 |
11319
a4b299e37909
mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents:
11318
diff
changeset
|
148 if not filesize or filesize < 0 or filesize % 1 ~= 0 then |
a4b299e37909
mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents:
11318
diff
changeset
|
149 return false, upload_errors.new("filesizefmt"); |
a4b299e37909
mod_http_file_share: Reject invalid file sizes
Kim Alvefur <zash@zash.se>
parents:
11318
diff
changeset
|
150 end |
11314
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
151 if filesize > file_size_limit then |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
152 return false, upload_errors.new("filesize"); |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
153 end |
7c8b02c5a335
mod_http_file_share: Add file size limit (default 10M)
Kim Alvefur <zash@zash.se>
parents:
11313
diff
changeset
|
154 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
155 if total_storage_usage + filesize > total_storage_limit then |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
156 module:log("warn", "Global storage quota reached, at %s / %s!", B(total_storage_usage), B(total_storage_limit)); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
157 return false, upload_errors.new("outofdisk"); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
158 end |
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
159 |
11346
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
160 local uploader_quota = get_daily_quota(uploader); |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
161 if uploader_quota + filesize > daily_quota then |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
162 return false, upload_errors.new("quota"); |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
163 end |
315faec1a920
mod_http_file_share: Add support for daily upload quotas.
Kim Alvefur <zash@zash.se>
parents:
11345
diff
changeset
|
164 |
11315
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
165 if not ( file_types:empty() or file_types:contains(filetype) or file_types:contains(filetype:gsub("/.*", "/*")) ) then |
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
166 return false, upload_errors.new("filetype"); |
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
167 end |
c52fcea39c8e
mod_http_file_share: Add file type filter
Kim Alvefur <zash@zash.se>
parents:
11314
diff
changeset
|
168 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
169 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
170 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
171 |
11350
3287dbdde33e
mod_http_file_share: Reorder arguments
Kim Alvefur <zash@zash.se>
parents:
11349
diff
changeset
|
172 function get_authz(slot, uploader, filename, filesize, filetype) |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
173 return create_jwt({ |
11501
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
174 -- token properties |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
175 sub = uploader; |
11501
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
176 |
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
177 -- slot properties |
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
178 slot = slot; |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
179 expires = expiry >= 0 and (os.time()+expiry) or nil; |
11501
2c9db2278fed
mod_http_file_share: Group related properties for readability
Kim Alvefur <zash@zash.se>
parents:
11500
diff
changeset
|
180 -- file properties |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
181 filename = filename; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
182 filesize = filesize; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
183 filetype = filetype; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
184 }); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
185 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
186 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
187 function get_url(slot, filename) |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
188 local base_url = external_base_url or module:http_url(); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
189 local slot_url = url.parse(base_url); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
190 slot_url.path = url.parse_path(slot_url.path or "/"); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
191 t_insert(slot_url.path, slot); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
192 if filename then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
193 t_insert(slot_url.path, filename); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
194 slot_url.path.is_directory = false; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
195 else |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
196 slot_url.path.is_directory = true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
197 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
198 slot_url.path = url.build_path(slot_url.path); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
199 return url.build(slot_url); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
200 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
201 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
202 function handle_slot_request(event) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
203 local stanza, origin = event.stanza, event.origin; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
204 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
205 local request = st.clone(stanza.tags[1], true); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
206 local filename = request.attr.filename; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
207 local filesize = tonumber(request.attr.size); |
11320
817cadf6be92
mod_http_file_share: Handle content-type being optional
Kim Alvefur <zash@zash.se>
parents:
11319
diff
changeset
|
208 local filetype = request.attr["content-type"] or "application/octet-stream"; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
209 local uploader = jid.bare(stanza.attr.from); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
210 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
211 local may, why_not = may_upload(uploader, filename, filesize, filetype); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
212 if not may then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
213 origin.send(st.error_reply(stanza, why_not)); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
214 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
215 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
216 |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
217 module:log("info", "Issuing upload slot to %s for %s", uploader, B(filesize)); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
218 local slot, storage_err = errors.coerce(uploads:append(nil, nil, request, os.time(), uploader)) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
219 if not slot then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
220 origin.send(st.error_reply(stanza, storage_err)); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
221 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
222 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
223 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
224 total_storage_usage = total_storage_usage + filesize; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
225 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
226 |
11349
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
227 local cached_quota = quota_cache:get(uploader); |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
228 if cached_quota and cached_quota.time > os.time()-86400 then |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
229 cached_quota.size = cached_quota.size + filesize; |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
230 quota_cache:set(uploader, cached_quota); |
a219001b449d
mod_http_file_share: Update cached value while it is reasonably fresh
Kim Alvefur <zash@zash.se>
parents:
11348
diff
changeset
|
231 end |
11348
f076199b4d38
mod_http_file_share: Cache quotas to avoid hitting storage
Kim Alvefur <zash@zash.se>
parents:
11347
diff
changeset
|
232 |
11350
3287dbdde33e
mod_http_file_share: Reorder arguments
Kim Alvefur <zash@zash.se>
parents:
11349
diff
changeset
|
233 local authz = get_authz(slot, uploader, filename, filesize, filetype); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
234 local slot_url = get_url(slot, filename); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
235 local upload_url = slot_url; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
236 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
237 local reply = st.reply(stanza) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
238 :tag("slot", { xmlns = namespace }) |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
239 :tag("get", { url = slot_url }):up() |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
240 :tag("put", { url = upload_url }) |
11322
4ade9810ce35
mod_http_file_share: Move Authorization type string
Kim Alvefur <zash@zash.se>
parents:
11321
diff
changeset
|
241 :text_tag("header", "Bearer "..authz, {name="Authorization"}) |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
242 :reset(); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
243 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
244 origin.send(reply); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
245 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
246 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
247 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
248 function handle_upload(event, path) -- PUT /upload/:slot |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
249 local request = event.request; |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
250 local upload_info = request.http_file_share_upload_info; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
251 |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
252 if not upload_info then -- Initial handling of request |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
253 local authz = request.headers.authorization; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
254 if authz then |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
255 authz = authz:match("^Bearer (.*)") |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
256 end |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
257 if not authz then |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
258 module:log("debug", "Missing or malformed Authorization header"); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
259 event.response.headers.www_authenticate = "Bearer"; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
260 return 401; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
261 end |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
262 local authed, authed_upload_info = verify_jwt(authz); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
263 if not authed then |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
264 module:log("debug", "Unauthorized or invalid token: %s, %q", authz, authed_upload_info); |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
265 return 401; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
266 end |
12722
cd993fd7b60d
mod_http_file_share: Use correct variable name (thanks riau.sni)
Matthew Wild <mwild1@gmail.com>
parents:
12708
diff
changeset
|
267 if not path or authed_upload_info.slot ~= path:match("^[^/]+") then |
cd993fd7b60d
mod_http_file_share: Use correct variable name (thanks riau.sni)
Matthew Wild <mwild1@gmail.com>
parents:
12708
diff
changeset
|
268 module:log("debug", "Invalid upload slot: %q, path: %q", authed_upload_info.slot, path); |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
269 return 400; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
270 end |
12722
cd993fd7b60d
mod_http_file_share: Use correct variable name (thanks riau.sni)
Matthew Wild <mwild1@gmail.com>
parents:
12708
diff
changeset
|
271 if request.headers.content_length and tonumber(request.headers.content_length) ~= authed_upload_info.filesize then |
12708
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
272 return 413; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
273 -- Note: We don't know the size if the upload is streamed in chunked encoding, |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
274 -- so we also check the final file size on completion. |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
275 end |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
276 upload_info = authed_upload_info; |
9953ac7b0c15
mod_http_file_share: Switch to new util.jwt API
Matthew Wild <mwild1@gmail.com>
parents:
12444
diff
changeset
|
277 request.http_file_share_upload_info = upload_info; |
11323
a853a018eede
mod_http_file_share: Validate file size early in HTTP PUT request
Kim Alvefur <zash@zash.se>
parents:
11322
diff
changeset
|
278 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
279 |
11325
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
280 local filename = get_filename(upload_info.slot, true); |
11324
494761f5d7da
mod_http_file_share: Use '.bin' file extension
Kim Alvefur <zash@zash.se>
parents:
11323
diff
changeset
|
281 |
11375
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
282 do |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
283 -- check if upload has been completed already |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
284 -- we want to allow retry of a failed upload attempt, but not after it's been completed |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
285 local f = io.open(filename, "r"); |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
286 if f then |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
287 f:close(); |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
288 return 409; |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
289 end |
6b687210975b
mod_http_file_share: Prevent attempt to upload again after completion
Kim Alvefur <zash@zash.se>
parents:
11374
diff
changeset
|
290 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
291 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
292 if not request.body_sink then |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
293 module:log("debug", "Preparing to receive upload into %q, expecting %s", filename, B(upload_info.filesize)); |
11500
21706a581b8a
mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents:
11499
diff
changeset
|
294 local fh, err = io.open(filename.."~", "w"); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
295 if not fh then |
11500
21706a581b8a
mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents:
11499
diff
changeset
|
296 module:log("error", "Could not open file for writing: %s", err); |
21706a581b8a
mod_http_file_share: Log error opening file for writing
Kim Alvefur <zash@zash.se>
parents:
11499
diff
changeset
|
297 return 500; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
298 end |
11865
77bbbd4263d7
mod_http_file_share: Silence luacheck warning
Kim Alvefur <zash@zash.se>
parents:
11864
diff
changeset
|
299 function event.response:on_destroy() -- luacheck: ignore 212/self |
11864
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
300 -- Clean up incomplete upload |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
301 if io.type(fh) == "file" then -- still open |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
302 fh:close(); |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
303 os.remove(filename.."~"); |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
304 end |
fceebfb28d86
mod_http_file_share: Clean up incomplete uploads
Kim Alvefur <zash@zash.se>
parents:
11857
diff
changeset
|
305 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
306 request.body_sink = fh; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
307 if request.body == false then |
11374
5b8aec0609f0
mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents:
11357
diff
changeset
|
308 if request.headers.expect == "100-continue" then |
5b8aec0609f0
mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents:
11357
diff
changeset
|
309 request.conn:write("HTTP/1.1 100 Continue\r\n\r\n"); |
5b8aec0609f0
mod_http_file_share: Support sending 100 Continue
Kim Alvefur <zash@zash.se>
parents:
11357
diff
changeset
|
310 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
311 return true; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
312 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
313 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
314 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
315 if request.body then |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
316 module:log("debug", "Complete upload available, %s", B(#request.body)); |
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
317 -- Small enough to have been uploaded already |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
318 local written, err = errors.coerce(request.body_sink:write(request.body)); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
319 if not written then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
320 return err; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
321 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
322 request.body = nil; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
323 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
324 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
325 if request.body_sink then |
11316
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
326 local final_size = request.body_sink:seek(); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
327 local uploaded, err = errors.coerce(request.body_sink:close()); |
11316
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
328 if final_size ~= upload_info.filesize then |
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
329 -- Could be too short as well, but we say the same thing |
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
330 uploaded, err = false, 413; |
ae0461b37fbe
mod_http_file_share: Verify final file size on completion of upload
Kim Alvefur <zash@zash.se>
parents:
11315
diff
changeset
|
331 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
332 if uploaded then |
11321
15ab878a7d23
mod_http_file_share: Add some logging
Kim Alvefur <zash@zash.se>
parents:
11320
diff
changeset
|
333 module:log("debug", "Upload of %q completed, %s", filename, B(final_size)); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
334 assert(os.rename(filename.."~", filename)); |
11355
89efa3f2966b
mod_http_file_share: Collect statistics of files uploaded
Kim Alvefur <zash@zash.se>
parents:
11350
diff
changeset
|
335 measure_uploads(final_size); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
336 |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
337 upload_cache:set(upload_info.slot, { |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
338 name = upload_info.filename; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
339 size = tostring(upload_info.filesize); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
340 type = upload_info.filetype; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
341 time = os.time(); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
342 }); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
343 return 201; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
344 else |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
345 assert(os.remove(filename.."~")); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
346 return err; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
347 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
348 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
349 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
350 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
351 |
11357
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
352 local download_cache_hit = module:measure("download_cache_hit", "rate"); |
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
353 local download_cache_miss = module:measure("download_cache_miss", "rate"); |
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
354 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
355 function handle_download(event, path) -- GET /uploads/:slot+filename |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
356 local request, response = event.request, event.response; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
357 local slot_id = path:match("^[^/]+"); |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
358 local basename, filetime, filetype, filesize; |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
359 local cached = upload_cache:get(slot_id); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
360 if cached then |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
361 module:log("debug", "Cache hit"); |
11357
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
362 download_cache_hit(); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
363 basename = cached.name; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
364 filesize = cached.size; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
365 filetype = cached.type; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
366 filetime = cached.time; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
367 upload_cache:set(slot_id, cached); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
368 -- TODO cache negative hits? |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
369 else |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
370 module:log("debug", "Cache miss"); |
11357
8cb2a64b15da
mod_http_file_share: Collect cache hit/miss statistics for downloads
Kim Alvefur <zash@zash.se>
parents:
11356
diff
changeset
|
371 download_cache_miss(); |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
372 local slot, when = errors.coerce(uploads:get(nil, slot_id)); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
373 if not slot then |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
374 module:log("debug", "uploads:get(%q) --> not-found, %s", slot_id, when); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
375 else |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
376 module:log("debug", "uploads:get(%q) --> %s, %d", slot_id, slot, when); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
377 basename = slot.attr.filename; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
378 filesize = slot.attr.size; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
379 filetype = slot.attr["content-type"]; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
380 filetime = when; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
381 upload_cache:set(slot_id, { |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
382 name = basename; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
383 size = slot.attr.size; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
384 type = filetype; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
385 time = when; |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
386 }); |
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
387 end |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
388 end |
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
389 if not basename then |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
390 return 404; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
391 end |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
392 local last_modified = os.date('!%a, %d %b %Y %H:%M:%S GMT', filetime); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
393 if request.headers.if_modified_since == last_modified then |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
394 return 304; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
395 end |
11325
76fc73d39092
mod_http_file_share: Factor out function for generating full filename
Kim Alvefur <zash@zash.se>
parents:
11324
diff
changeset
|
396 local filename = get_filename(slot_id); |
11493
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
397 local handle, ferr = io.open(filename); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
398 if not handle then |
11499
a8cbbbb1f165
mod_http_file_share: Fix logging of error opening file
Kim Alvefur <zash@zash.se>
parents:
11496
diff
changeset
|
399 module:log("error", "Could not open file for reading: %s", ferr); |
11493
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
400 -- This can be because the upload slot wasn't used, or the file disappeared |
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
401 -- somehow, or permission issues. |
77f2d45799ed
mod_http_file_share: Fix reporting of missing files
Kim Alvefur <zash@zash.se>
parents:
11491
diff
changeset
|
402 return 410; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
403 end |
11333
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
404 |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
405 local request_range = request.headers.range; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
406 local response_range; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
407 if request_range then |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
408 local range_start, range_end = request_range:match("^bytes=(%d+)%-(%d*)$") |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
409 -- Only support resumption, ie ranges from somewhere in the middle until the end of the file. |
11568
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
410 if (range_start and range_start ~= "0") and (range_end == "" or range_end == filesize) then |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
411 local pos, size = tonumber(range_start), tonumber(filesize); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
412 local new_pos = pos < size and handle:seek("set", pos); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
413 if new_pos and new_pos < size then |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
414 response_range = "bytes "..range_start.."-"..filesize.."/"..filesize; |
11568
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
415 filesize = string.format("%d", size-pos); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
416 else |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
417 handle:close(); |
d5360307a99d
mod_http_file_share: Handle out of bounds Range request
Kim Alvefur <zash@zash.se>
parents:
11564
diff
changeset
|
418 return 416; |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
419 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
420 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
421 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
422 |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
423 |
11402
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
424 if not filetype then |
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
425 filetype = "application/octet-stream"; |
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
426 end |
11333
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
427 local disposition = "attachment"; |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
428 if safe_types:contains(filetype) or safe_types:contains(filetype:gsub("/.*", "/*")) then |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
429 disposition = "inline"; |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
430 end |
f80056b97cf0
mod_http_file_share: Serve configurable set of safe mime types inline (thanks jonas’)
Kim Alvefur <zash@zash.se>
parents:
11332
diff
changeset
|
431 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
432 response.headers.last_modified = last_modified; |
11331
7a915fa49373
mod_http_file_share: Extract all file properties into variables earlier
Kim Alvefur <zash@zash.se>
parents:
11330
diff
changeset
|
433 response.headers.content_length = filesize; |
11402
a3be7b3cf1e1
mod_http_file_share: Fix traceback on missing file-type
Kim Alvefur <zash@zash.se>
parents:
11398
diff
changeset
|
434 response.headers.content_type = filetype; |
12227
88958c0ecab3
mod_http_file_share: Use alternate syntax for filename in Content-Disposition
Kim Alvefur <zash@zash.se>
parents:
12179
diff
changeset
|
435 response.headers.content_disposition = string.format("%s; filename*=UTF-8''%s", disposition, urlencode(basename)); |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
436 |
11564
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
437 if response_range then |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
438 response.status_code = 206; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
439 response.headers.content_range = response_range; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
440 end |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
441 response.headers.accept_ranges = "bytes"; |
60e31c9ece57
mod_http_file_share: Support download resumption via Range requests
Kim Alvefur <zash@zash.se>
parents:
11503
diff
changeset
|
442 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
443 response.headers.cache_control = "max-age=31556952, immutable"; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
444 response.headers.content_security_policy = "default-src 'none'; frame-ancestors 'none';" |
11327
6f2b69469060
mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents:
11326
diff
changeset
|
445 response.headers.strict_transport_security = "max-age=31556952"; |
6f2b69469060
mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents:
11326
diff
changeset
|
446 response.headers.x_content_type_options = "nosniff"; |
11611
a6d1131ac833
mod_http_file_share: Update comment about x-frame-options
Kim Alvefur <zash@zash.se>
parents:
11594
diff
changeset
|
447 response.headers.x_frame_options = "DENY"; -- COMPAT IE missing support for CSP frame-ancestors |
11327
6f2b69469060
mod_http_file_share: More security headers
Kim Alvefur <zash@zash.se>
parents:
11326
diff
changeset
|
448 response.headers.x_xss_protection = "1; mode=block"; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
449 |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
450 return response:send_file(handle); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
451 end |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
452 |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
453 if expiry >= 0 and not external_base_url then |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
454 -- TODO HTTP DELETE to the external endpoint? |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
455 local array = require "util.array"; |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
456 local async = require "util.async"; |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
457 local ENOENT = require "util.pposix".ENOENT; |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
458 |
11329
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
459 local function sleep(t) |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
460 local wait, done = async.waiter(); |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
461 module:add_timer(t, done) |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
462 wait(); |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
463 end |
2a431d3ad8f1
mod_http_file_share: Insert pauses to avoid blocknig for long periods
Kim Alvefur <zash@zash.se>
parents:
11328
diff
changeset
|
464 |
11802
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
465 local prune_start = module:measure("prune", "times"); |
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
466 |
12008
c01532ae6a3b
mod_http_file_share: Fix to take retention time into account
Kim Alvefur <zash@zash.se>
parents:
12007
diff
changeset
|
467 module:daily("Remove expired files", function(_, current_time) |
11802
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
468 local prune_done = prune_start(); |
12008
c01532ae6a3b
mod_http_file_share: Fix to take retention time into account
Kim Alvefur <zash@zash.se>
parents:
12007
diff
changeset
|
469 local boundary_time = (current_time or os.time()) - expiry; |
12004
a2a0e00cbd24
mod_http_file_share: Back out 876e1b6d6ae4
Kim Alvefur <zash@zash.se>
parents:
12003
diff
changeset
|
470 local iter, total = assert(uploads:find(nil, {["end"] = boundary_time; total = true})); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
471 |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
472 if total == 0 then |
11343
f125ac529c22
mod_http_file_share: Clarify log message
Kim Alvefur <zash@zash.se>
parents:
11336
diff
changeset
|
473 module:log("info", "No expired uploaded files to prune"); |
11356
43e5429ab355
mod_http_file_share: Measure how long it takes to prune expired files
Kim Alvefur <zash@zash.se>
parents:
11355
diff
changeset
|
474 prune_done(); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
475 return; |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
476 end |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
477 |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
478 module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time)); |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
479 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
480 |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
481 local obsolete_uploads = array(); |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
482 local num_expired = 0; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
483 local size_sum = 0; |
11993
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
484 local problem_deleting = false; |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
485 for slot_id, slot_info in iter do |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
486 num_expired = num_expired + 1; |
11332
3e0dcdf6283e
mod_http_file_share: Cache file metadata
Kim Alvefur <zash@zash.se>
parents:
11331
diff
changeset
|
487 upload_cache:set(slot_id, nil); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
488 local filename = get_filename(slot_id); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
489 local deleted, err, errno = os.remove(filename); |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
490 if deleted or errno == ENOENT then -- removed successfully or it was already gone |
11993
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
491 size_sum = size_sum + tonumber(slot_info.attr.size); |
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
492 obsolete_uploads:push(slot_id); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
493 else |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
494 module:log("error", "Could not prune expired file %q: %s", filename, err); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
495 problem_deleting = true; |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
496 end |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
497 if num_expired % 100 == 0 then sleep(0.1); end |
11993
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
498 end |
aa60f4353001
mod_http_file_share: Merge file expiry loops
Kim Alvefur <zash@zash.se>
parents:
11992
diff
changeset
|
499 |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
500 -- obsolete_uploads now contains slot ids for which the files have been |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
501 -- removed and that needs to be cleared from the database |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
502 |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
503 local deletion_query = {["end"] = boundary_time}; |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
504 if not problem_deleting then |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
505 module:log("info", "All (%d, %s) expired files successfully pruned", num_expired, B(size_sum)); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
506 -- we can delete based on time |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
507 else |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
508 module:log("warn", "%d out of %d expired files could not be pruned", num_expired-#obsolete_uploads, num_expired); |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
509 -- we'll need to delete only those entries where the files were |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
510 -- successfully removed, and then try again with the failed ones. |
11405
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
511 -- eventually the admin ought to notice and fix the permissions or |
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
512 -- whatever the problem is. |
ce8291e89d67
mod_http_file_share: Remove correct entries when not all expired files were deleted
Kim Alvefur <zash@zash.se>
parents:
11402
diff
changeset
|
513 deletion_query = {ids = obsolete_uploads}; |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
514 end |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
515 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
516 total_storage_usage = total_storage_usage - size_sum; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
517 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
518 persist_stats:set(nil, "total", total_storage_usage); |
11994
f9b2325f6b50
mod_http_file_share: Keep global storage use accurate longer.
Kim Alvefur <zash@zash.se>
parents:
11993
diff
changeset
|
519 |
11406
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
520 if #obsolete_uploads == 0 then |
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
521 module:log("debug", "No metadata to remove"); |
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
522 else |
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
523 local removed, err = uploads:delete(nil, deletion_query); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
524 |
12006
62a466e60515
mod_http_file_share: Rename variable for clarity
Kim Alvefur <zash@zash.se>
parents:
12005
diff
changeset
|
525 if removed == true or removed == num_expired or removed == #obsolete_uploads then |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
526 module:log("debug", "Expired upload metadata pruned successfully"); |
11406
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
527 else |
12007
dc72581e04ff
mod_http_file_share: Improve consistency of terminology in logging
Kim Alvefur <zash@zash.se>
parents:
12006
diff
changeset
|
528 module:log("error", "Problem removing metadata for expired files: %s", err); |
11406
9d6545a7d483
mod_http_file_share: Skip removal of nothing
Kim Alvefur <zash@zash.se>
parents:
11405
diff
changeset
|
529 end |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
530 end |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
531 |
11356
43e5429ab355
mod_http_file_share: Measure how long it takes to prune expired files
Kim Alvefur <zash@zash.se>
parents:
11355
diff
changeset
|
532 prune_done(); |
11328
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
533 end); |
ceaa3cebf28b
mod_http_file_share: Add support for removing old files (default 2 weeks)
Kim Alvefur <zash@zash.se>
parents:
11327
diff
changeset
|
534 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
535 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
536 local summary_start = module:measure("summary", "times"); |
11802
3d411720e719
mod_http_file_share: Fix measuring how long periodic task take
Kim Alvefur <zash@zash.se>
parents:
11794
diff
changeset
|
537 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
538 module:weekly("Calculate total storage usage", function() |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
539 local summary_done = summary_start(); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
540 local iter = assert(uploads:find(nil)); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
541 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
542 local count, sum = 0, 0; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
543 for _, file in iter do |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
544 sum = sum + tonumber(file.attr.size); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
545 count = count + 1; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
546 end |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
547 |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
548 module:log("info", "Uploaded files total: %s in %d files", B(sum), count); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
549 if persist_stats:set(nil, "total", sum) then |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
550 total_storage_usage = sum; |
12179
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
551 else |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
552 total_storage_usage = unknown; |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
553 end |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
554 module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit)); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
555 summary_done(); |
5e68635cdc2c
mod_http_file_share: Always measure total disk usage for statistics!
Kim Alvefur <zash@zash.se>
parents:
12008
diff
changeset
|
556 end); |
11781
9c23e7c8a67a
mod_http_file_share: Add optional global quota on total storage usage
Kim Alvefur <zash@zash.se>
parents:
11611
diff
changeset
|
557 |
11495
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
558 -- Reachable from the console |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
559 function check_files(query) |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
560 local issues = {}; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
561 local iter = assert(uploads:find(nil, query)); |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
562 for slot_id, file in iter do |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
563 local filename = get_filename(slot_id); |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
564 local size, err = lfs.attributes(filename, "size"); |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
565 if not size then |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
566 issues[filename] = err; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
567 elseif tonumber(file.attr.size) ~= size then |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
568 issues[filename] = "file size mismatch"; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
569 end |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
570 end |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
571 |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
572 return next(issues) == nil, issues; |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
573 end |
6d3f84148729
mod_http_file_share: Add internal command to check files consistency
Kim Alvefur <zash@zash.se>
parents:
11493
diff
changeset
|
574 |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
575 module:hook("iq-get/host/urn:xmpp:http:upload:0:request", handle_slot_request); |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
576 |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
577 if not external_base_url then |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
578 module:provides("http", { |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
579 streaming_uploads = true; |
11398
a1f26d17d70f
mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents:
11394
diff
changeset
|
580 cors = { |
12444
b33558969b3e
mod_http (and dependent modules): Make CORS opt-in by default (fixes #1731)
Matthew Wild <mwild1@gmail.com>
parents:
12227
diff
changeset
|
581 enabled = true; |
11398
a1f26d17d70f
mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents:
11394
diff
changeset
|
582 credentials = true; |
11857
e080d6aa0b3b
mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents:
11853
diff
changeset
|
583 headers = { |
e080d6aa0b3b
mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents:
11853
diff
changeset
|
584 Authorization = true; |
e080d6aa0b3b
mod_http_file_share: Allow 'Authorization' header via CORS (thanks kawaii)
Kim Alvefur <zash@zash.se>
parents:
11853
diff
changeset
|
585 }; |
11398
a1f26d17d70f
mod_http_file_share: Allow credentials via CORS (needed for auth token)
Kim Alvefur <zash@zash.se>
parents:
11394
diff
changeset
|
586 }; |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
587 route = { |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
588 ["PUT /*"] = handle_upload; |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
589 ["GET /*"] = handle_download; |
11394
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
590 ["GET /"] = function (event) |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
591 return prosody.events.fire_event("http-message", { |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
592 response = event.response; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
593 --- |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
594 title = "Prosody HTTP Upload endpoint"; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
595 message = "This is where files will be uploaded to, and served from."; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
596 warning = not (event.request.secure) and "This endpoint is not considered secure!" or nil; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
597 }) or "This is the Prosody HTTP Upload endpoint."; |
420787340339
mod_http_file_share: Return a message from the base URL
Kim Alvefur <zash@zash.se>
parents:
11375
diff
changeset
|
598 end |
11309
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
599 } |
b59aed75dc5e
mod_http_file_share: Let's write another XEP-0363 implementation
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
600 }); |
11310
d1a0f2e918c0
mod_http_file_share: Add support for external file upload service
Kim Alvefur <zash@zash.se>
parents:
11309
diff
changeset
|
601 end |