Software /
code /
prosody-modules
Annotate
mod_pubsub_feeds/mod_pubsub_feeds.lua @ 5571:ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
This should ensure that an entry that has a publish timestmap after the
previously oldest post, but before the time of the last poll check, is
published to the node.
Previously if an entry would be skipped if it was published at 13:00
with a timestamp of 12:30, where the last poll was at 12:45.
For feeds that lack a timestamp, it now looks for the first post that is
not published, assuming that the feed is in reverse chronological order,
then iterates back up from there.
author | Kim Alvefur <zash@zash.se> |
---|---|
date | Sun, 25 Jun 2023 16:27:55 +0200 |
parent | 5570:f93b1fc1aa31 |
child | 5572:fd1c535dcb92 |
rev | line source |
---|---|
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
1 -- Fetches Atom feeds and publishes to PubSub nodes |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
2 -- |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
3 -- Config: |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
4 -- Component "pubsub.example.com" "pubsub" |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
5 -- modules_enabled = { |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
6 -- "pubsub_feeds"; |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
7 -- } |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
8 -- feeds = { -- node -> url |
279
aa0df3db4901
mod_pubsub_feed: Wrap entry in a item element.
Kim Alvefur <zash@zash.se>
parents:
278
diff
changeset
|
9 -- prosody_blog = "http://blog.prosody.im/feed/atom.xml"; |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
10 -- } |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
11 -- feed_pull_interval = 20 -- minutes |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
12 -- |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
13 -- Reference |
1454
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
14 -- http://pubsubhubbub.googlecode.com/svn/trunk/pubsubhubbub-core-0.4.html |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
15 |
792
db27f07342d4
mod_pubsub_feeds: Use reference to mod_pubsub from module:depends
Kim Alvefur <zash@zash.se>
parents:
765
diff
changeset
|
16 local pubsub = module:depends"pubsub"; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
17 |
3047
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2645
diff
changeset
|
18 local time = os.time; |
1325
b21236b6b8d8
Backed out changeset 853a382c9bd6
Kim Alvefur <zash@zash.se>
parents:
1324
diff
changeset
|
19 local dt_parse, dt_datetime = require "util.datetime".parse, require "util.datetime".datetime; |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
20 local uuid = require "util.uuid".generate; |
1107
04cf0b75fe2a
mod_pubsub_feeds: Use HMAC-SHA1 from util.hashes (util.hmac was deprecated)
Kim Alvefur <zash@zash.se>
parents:
983
diff
changeset
|
21 local hmac_sha1 = require "util.hashes".hmac_sha1; |
2137
0fe725b50f77
mod_pubsub_feeds.lua: util.xml not uit.xml
Michael Töglhofer <michael@toeglhofer.net>
parents:
2134
diff
changeset
|
22 local parse_xml = require "util.xml".parse; |
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
23 local st = require "util.stanza"; |
2132
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
24 local translate_rss = module:require("feeds").translate_rss; |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
25 |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
26 local xmlns_atom = "http://www.w3.org/2005/Atom"; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
27 |
2132
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
28 local function parse_feed(data) |
4883
68bd8ae9b827
mod_pubsub_feeds: Allow but ignore XML comments and processing instructions
Kim Alvefur <zash@zash.se>
parents:
4282
diff
changeset
|
29 local feed, err = parse_xml(data, { allow_processing_instructions = true; allow_comments = true }); |
2132
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
30 if not feed then return feed, err; end |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
31 if feed.attr.xmlns == xmlns_atom then |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
32 return feed; |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
33 elseif feed.attr.xmlns == nil and feed.name == "rss" then |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
34 return translate_rss(feed); |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
35 end |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
36 return nil, "unsupported-format"; |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
37 end |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
38 |
5569
1f8c9e27b625
mod_pubsub_feeds: Disable WebSub (formerly PubSubHubbub) by default
Kim Alvefur <zash@zash.se>
parents:
4883
diff
changeset
|
39 local use_pubsubhubub = module:get_option_boolean("use_pubsubhubub", false); |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
40 if use_pubsubhubub then |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
41 module:depends"http"; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
42 end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
43 |
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
44 local http = require "net.http"; |
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
45 local formdecode = http.formdecode; |
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
46 local formencode = http.formencode; |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
47 |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
48 local feed_list = module:shared("feed_list"); |
5570
f93b1fc1aa31
mod_pubsub_feeds: Add new interval setting in seconds (old still works)
Kim Alvefur <zash@zash.se>
parents:
5569
diff
changeset
|
49 local legacy_refresh_interval = module:get_option_number("feed_pull_interval", 15); |
f93b1fc1aa31
mod_pubsub_feeds: Add new interval setting in seconds (old still works)
Kim Alvefur <zash@zash.se>
parents:
5569
diff
changeset
|
50 local refresh_interval = module:get_option_number("feed_pull_interval_seconds", legacy_refresh_interval*60); |
2401
7a1625a84624
mod_pubsub_feeds: Ask for leases that expire after one day (Years after testing this module, I was still subscribed to a GNU Social instance)
Kim Alvefur <zash@zash.se>
parents:
2384
diff
changeset
|
51 local lease_length = tostring(math.floor(module:get_option_number("feed_lease_length", 86400))); |
475
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
52 |
794
5a8fa70faa13
mod_pubsub_feeds: Remove config-reloaded hook, do setup in module.load() instead
Kim Alvefur <zash@zash.se>
parents:
793
diff
changeset
|
53 function module.load() |
2377
02532f28b207
mod_pubsub_feeds: Remove default feeds
Kim Alvefur <zash@zash.se>
parents:
2137
diff
changeset
|
54 local config = module:get_option("feeds", { }); |
797
68a067d5ed3b
mod_pubsub_feeds: util.pubsub methods return bool, data
Kim Alvefur <zash@zash.se>
parents:
796
diff
changeset
|
55 local ok, nodes = pubsub.service:get_nodes(true); |
68a067d5ed3b
mod_pubsub_feeds: util.pubsub methods return bool, data
Kim Alvefur <zash@zash.se>
parents:
796
diff
changeset
|
56 if not ok then nodes = {}; end |
475
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
57 local new_feed_list = {}; |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
58 for node, url in pairs(config) do |
765
82d292e4f30f
mod_pubsub_feeds: Use URL as node name if the config index is numeric
Kim Alvefur <zash@zash.se>
parents:
763
diff
changeset
|
59 if type(node) == "number" then |
82d292e4f30f
mod_pubsub_feeds: Use URL as node name if the config index is numeric
Kim Alvefur <zash@zash.se>
parents:
763
diff
changeset
|
60 node = url; |
82d292e4f30f
mod_pubsub_feeds: Use URL as node name if the config index is numeric
Kim Alvefur <zash@zash.se>
parents:
763
diff
changeset
|
61 end |
475
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
62 new_feed_list[node] = true; |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
63 if not feed_list[node] then |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
64 feed_list[node] = { url = url; node = node; last_update = 0 }; |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
65 else |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
66 feed_list[node].url = url; |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
67 end |
795
c7f507a190a0
mod_pubsub_feeds: Check if nodes exists in pubsub, make sure they get restored if not. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
794
diff
changeset
|
68 if not nodes[node] then |
c7f507a190a0
mod_pubsub_feeds: Check if nodes exists in pubsub, make sure they get restored if not. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
794
diff
changeset
|
69 feed_list[node].last_update = 0; |
c7f507a190a0
mod_pubsub_feeds: Check if nodes exists in pubsub, make sure they get restored if not. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
794
diff
changeset
|
70 end |
475
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
71 end |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
72 for node in pairs(feed_list) do |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
73 if not new_feed_list[node] then |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
74 feed_list[node] = nil; |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
75 end |
db5702bb9e41
mod_pubsub_feed: Dynamicaly reloadable config.
Kim Alvefur <zash@zash.se>
parents:
461
diff
changeset
|
76 end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
77 end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
78 |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
79 function update_entry(item) |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
80 local node = item.node; |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
81 module:log("debug", "parsing %d bytes of data in node %s", #item.data or 0, node) |
3281
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3280
diff
changeset
|
82 local feed, err = parse_feed(item.data); |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3280
diff
changeset
|
83 if not feed then |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3280
diff
changeset
|
84 module:log("error", "Could not parse feed %q: %s", item.url, err); |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3280
diff
changeset
|
85 module:log("debug", "Feed data:\n%s\n.", item.data); |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3280
diff
changeset
|
86 return; |
27cc66bf918b
mod_pubsub_feeds: Handle feed parsing error (thanks apollo13)
Kim Alvefur <zash@zash.se>
parents:
3280
diff
changeset
|
87 end |
2379
bc6e63ab2a7c
mod_pubsub_feeds: Iterate over feed backwards due to most feeds being in reverse chronological order
Kim Alvefur <zash@zash.se>
parents:
2377
diff
changeset
|
88 local entries = {}; |
2132
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
89 for entry in feed:childtags("entry") do |
2379
bc6e63ab2a7c
mod_pubsub_feeds: Iterate over feed backwards due to most feeds being in reverse chronological order
Kim Alvefur <zash@zash.se>
parents:
2377
diff
changeset
|
90 table.insert(entries, entry); |
bc6e63ab2a7c
mod_pubsub_feeds: Iterate over feed backwards due to most feeds being in reverse chronological order
Kim Alvefur <zash@zash.se>
parents:
2377
diff
changeset
|
91 end |
2383
18323c6aa133
mod_pubsub_feeds: Halt feed parsing when we see a node we already know about
Kim Alvefur <zash@zash.se>
parents:
2382
diff
changeset
|
92 local ok, items = pubsub.service:get_items(node, true); |
2382
f159ad7ac18a
mod_pubsub_feeds: Check if node exists earlier and attempt to create it if not
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
93 if not ok then |
f159ad7ac18a
mod_pubsub_feeds: Check if node exists earlier and attempt to create it if not
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
94 local ok, err = pubsub.service:create(node, true); |
f159ad7ac18a
mod_pubsub_feeds: Check if node exists earlier and attempt to create it if not
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
95 if not ok then |
f159ad7ac18a
mod_pubsub_feeds: Check if node exists earlier and attempt to create it if not
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
96 module:log("error", "Could not create node %s: %s", node, err); |
f159ad7ac18a
mod_pubsub_feeds: Check if node exists earlier and attempt to create it if not
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
97 return; |
f159ad7ac18a
mod_pubsub_feeds: Check if node exists earlier and attempt to create it if not
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
98 end |
2383
18323c6aa133
mod_pubsub_feeds: Halt feed parsing when we see a node we already know about
Kim Alvefur <zash@zash.se>
parents:
2382
diff
changeset
|
99 items = {}; |
2382
f159ad7ac18a
mod_pubsub_feeds: Check if node exists earlier and attempt to create it if not
Kim Alvefur <zash@zash.se>
parents:
2381
diff
changeset
|
100 end |
5571
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
101 |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
102 local start_from = #entries; |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
103 for i, entry in ipairs(entries) do |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
104 local id = entry:get_child_text("id"); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
105 if not id then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
106 local link = entry:get_child("link"); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
107 if link then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
108 module:log("debug", "Feed %q item %s is missing an id, using <link> instead", feed.url, entry:top_tag()); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
109 id = link and link.attr.href; |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
110 else |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
111 module:log("debug", "Feed %q item %s is missing an id, using a HMAC of the item instead", feed.url, entry:top_tag()); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
112 id = feed.url .. "#" .. hmac_sha1(feed.url, tostring(entry), true) .. "@" .. dt_datetime(timestamp); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
113 end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
114 entry:text_tag("id", id); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
115 end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
116 |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
117 if items[id] then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
118 -- This should be the first item that we already have. |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
119 start_from = i-1; |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
120 break |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
121 end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
122 end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
123 |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
124 for i = start_from, 1, -1 do -- Feeds are usually in reverse order |
2381
a4614a329bdd
mod_pubsub_feeds: Get entry from list of entries (thanks Link Mauve)
Kim Alvefur <zash@zash.se>
parents:
2380
diff
changeset
|
125 local entry = entries[i]; |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
126 entry.attr.xmlns = xmlns_atom; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
127 |
5571
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
128 local id = entry:get_child_text("id"); |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
129 |
5571
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
130 local timestamp = dt_parse(entry:get_child_text("published")); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
131 if not timestamp then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
132 timestamp = time(); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
133 entry:text_tag("published", dt_datetime(timestamp)); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
134 end |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
135 |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
136 if not timestamp or not item.last_update or timestamp > item.last_update then |
5571
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
137 local xitem = st.stanza("item", { id = id, xmlns = "http://jabber.org/protocol/pubsub" }):add_child(entry); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
138 -- TODO Put data from /feed into item/source |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
139 |
5571
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
140 local ok, err = pubsub.service:publish(node, true, id, xitem); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
141 if not ok then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
142 module:log("error", "Publishing to node %s failed: %s", node, err); |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
143 elseif timestamp then |
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
144 item.last_update = timestamp; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
145 end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
146 end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
147 end |
1343
7dbde05b48a9
all the things: Remove trailing whitespace
Florian Zeitz <florob@babelmonkeys.de>
parents:
1325
diff
changeset
|
148 |
1457
720ff25d94e6
mod_pubsub_feeds: Check that lease time exists before comparing
Kim Alvefur <zash@zash.se>
parents:
1456
diff
changeset
|
149 if item.lease_expires and item.lease_expires > time() then |
1455
13e359c48b5b
mod_pubsub_feeds: Fix refreshing of subscriptions
Kim Alvefur <zash@zash.se>
parents:
1454
diff
changeset
|
150 item.subscription = nil; |
13e359c48b5b
mod_pubsub_feeds: Fix refreshing of subscriptions
Kim Alvefur <zash@zash.se>
parents:
1454
diff
changeset
|
151 item.lease_expires = nil; |
13e359c48b5b
mod_pubsub_feeds: Fix refreshing of subscriptions
Kim Alvefur <zash@zash.se>
parents:
1454
diff
changeset
|
152 end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
153 if use_pubsubhubub and not item.subscription then |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
154 --module:log("debug", "check if %s has a hub", item.node); |
2132
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
155 for link in feed:childtags("link") do |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
156 if link.attr.rel == "hub" then |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
157 item.hub = link.attr.href; |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
158 module:log("debug", "Node %s has a hub: %s", item.node, item.hub); |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
159 return subscribe(item); |
b149ea428b81
mod_pubsub_feeds: Switch to use util.xml for parsing feeds and include RSS to Atom translation code from lua-feeds
Kim Alvefur <zash@zash.se>
parents:
1457
diff
changeset
|
160 end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
161 end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
162 end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
163 end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
164 |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
165 function fetch(item, callback) -- HTTP Pull |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
166 local headers = { }; |
3047
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2645
diff
changeset
|
167 if item.data and item.etag then |
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2645
diff
changeset
|
168 headers["If-None-Match"] = item.etag; |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
169 end |
3047
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2645
diff
changeset
|
170 http.request(item.url, { headers = headers }, function(data, code, resp) |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
171 if code == 200 then |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
172 item.data = data; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
173 if callback then callback(item) end |
3047
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2645
diff
changeset
|
174 if resp.headers then |
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2645
diff
changeset
|
175 item.etag = resp.headers.etag |
7c55f05327a2
mod_pubsub_feeds: Use ETag instead of problematic If-Modified-Since
Kim Alvefur <zash@zash.se>
parents:
2645
diff
changeset
|
176 end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
177 elseif code == 304 then |
5571
ca3c2d11823c
mod_pubsub_feeds: Track latest timestamp seen in feeds instead of last poll
Kim Alvefur <zash@zash.se>
parents:
5570
diff
changeset
|
178 module:log("debug", "No updates to %q", item.url); |
3279
e6b25054c110
mod_pubsub_feeds: Log a message for HTTP 301 redirects
Kim Alvefur <zash@zash.se>
parents:
3278
diff
changeset
|
179 elseif code == 301 and resp.headers.location then |
e6b25054c110
mod_pubsub_feeds: Log a message for HTTP 301 redirects
Kim Alvefur <zash@zash.se>
parents:
3278
diff
changeset
|
180 module:log("info", "Feed %q has moved to %q", item.url, resp.headers.location); |
3280
645c4f14e03d
mod_pubsub_feeds: Fix syntax error
Kim Alvefur <zash@zash.se>
parents:
3279
diff
changeset
|
181 elseif code <= 100 then |
3278
958f0b8b502c
mod_pubsub_feeds: Log non-HTTP errors like certificate problems etc
Kim Alvefur <zash@zash.se>
parents:
3277
diff
changeset
|
182 module:log("error", "Error fetching %q: %q[%d]", item.url, data, code); |
3277
78b11fb291a2
mod_pubsub_feeds: Log a debug message for status codes not handled
Kim Alvefur <zash@zash.se>
parents:
3242
diff
changeset
|
183 else |
78b11fb291a2
mod_pubsub_feeds: Log a debug message for status codes not handled
Kim Alvefur <zash@zash.se>
parents:
3242
diff
changeset
|
184 module:log("debug", "Unhandled status code %d when fetching %q", code, item.url); |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
185 end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
186 end); |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
187 end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
188 |
1453
7b53cfc6ba8d
mod_pubsub_feeds: Fix refresh_feeds() allways refreshing all feeds
Kim Alvefur <zash@zash.se>
parents:
1343
diff
changeset
|
189 function refresh_feeds(now) |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
190 --module:log("debug", "Refreshing feeds"); |
2134
bd7744df0b4a
mod_pubsub_feeds: Remove unused imports [luacheck]
Kim Alvefur <zash@zash.se>
parents:
2132
diff
changeset
|
191 for _, item in pairs(feed_list) do |
1453
7b53cfc6ba8d
mod_pubsub_feeds: Fix refresh_feeds() allways refreshing all feeds
Kim Alvefur <zash@zash.se>
parents:
1343
diff
changeset
|
192 if item.subscription ~= "subscribe" and item.last_update + refresh_interval < now then |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
193 --module:log("debug", "checking %s", item.node); |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
194 fetch(item, update_entry); |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
195 end |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
196 end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
197 return refresh_interval; |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
198 end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
199 |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
200 local function format_url(node) |
2134
bd7744df0b4a
mod_pubsub_feeds: Remove unused imports [luacheck]
Kim Alvefur <zash@zash.se>
parents:
2132
diff
changeset
|
201 return module:http_url(nil, "/callback") .. "?" .. formencode({ node = node }); |
1343
7dbde05b48a9
all the things: Remove trailing whitespace
Florian Zeitz <florob@babelmonkeys.de>
parents:
1325
diff
changeset
|
202 end |
401
c85397063eca
mod_pubsub_feed: Try harder to figure out ones callback URL.
Kim Alvefur <zash@zash.se>
parents:
400
diff
changeset
|
203 |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
204 function subscribe(feed, want) |
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
205 want = want or "subscribe"; |
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
206 feed.secret = feed.secret or uuid(); |
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
207 local body = formencode{ |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
208 ["hub.callback"] = format_url(feed.node); |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
209 ["hub.mode"] = want; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
210 ["hub.topic"] = feed.url; |
1454
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
211 ["hub.verify"] = "async"; -- COMPAT this is REQUIRED in the 0.3 draft but removed in 0.4 |
325
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
212 ["hub.secret"] = feed.secret; |
2401
7a1625a84624
mod_pubsub_feeds: Ask for leases that expire after one day (Years after testing this module, I was still subscribed to a GNU Social instance)
Kim Alvefur <zash@zash.se>
parents:
2384
diff
changeset
|
213 ["hub.lease_seconds"] = lease_length; |
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
214 }; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
215 |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
216 --module:log("debug", "subscription request, body: %s", body); |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
217 |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
218 --FIXME The subscription states and related stuff |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
219 feed.subscription = want; |
983
ac9bf3fcbcfe
mod_pubsub_feeds, mod_sms_clickatell, mod_twitter: Update for net.http API change in prosody:e3b9dc9dd940
Matthew Wild <mwild1@gmail.com>
parents:
797
diff
changeset
|
220 http.request(feed.hub, { body = body }, function(data, code) |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
221 module:log("debug", "subscription to %s submitted, status %s", feed.node, tostring(code)); |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
222 if code >= 400 then |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
223 module:log("error", "There was something wrong with our subscription request, body: %s", tostring(data)); |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
224 feed.subscription = "failed"; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
225 end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
226 end); |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
227 end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
228 |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
229 function handle_http_request(event) |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
230 local request = event.request; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
231 local method = request.method; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
232 local body = request.body; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
233 |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
234 local query = request.url.query or {}; --FIXME |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
235 if query and type(query) == "string" then |
403
fc62b26dfdf6
mod_pubsub_feed: Major cleanup, and use newer APIs. (Thanks Maranda)
Kim Alvefur <zash@zash.se>
parents:
402
diff
changeset
|
236 query = formdecode(query); |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
237 end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
238 |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
239 local feed = feed_list[query.node]; |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
240 if not feed then |
2422
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2421
diff
changeset
|
241 if query["hub.mode"] == "unsubscribe" then |
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2421
diff
changeset
|
242 -- Unsubscribe from unknown feed |
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2421
diff
changeset
|
243 module:log("debug", "Unsubscribe from unknown feed %s -- %s", query["hub.topic"], formencode(query)); |
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2421
diff
changeset
|
244 return query["hub.challenge"]; |
57235bce06fc
mod_pubsub_feeds: Always approve unsubscription from unknown feeds
Kim Alvefur <zash@zash.se>
parents:
2421
diff
changeset
|
245 end |
2421
a9add2d1adf5
mod_pubsub_feeds: Log pushes for unknown feeds to aid in manual unsubscription from them
Kim Alvefur <zash@zash.se>
parents:
2401
diff
changeset
|
246 module:log("debug", "Push for unknown feed %s -- %s", query["hub.topic"], formencode(query)); |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
247 return 404; |
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
248 end |
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
249 |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
250 if method == "GET" then |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
251 if query.node then |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
252 if query["hub.topic"] ~= feed.url then |
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
253 module:log("debug", "Invalid topic: %s", tostring(query["hub.topic"])) |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
254 return 404 |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
255 end |
1456
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
256 if query["hub.mode"] == "denied" then |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
257 module:log("info", "Subscription denied: %s", tostring(query["hub.reason"] or "No reason given")) |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
258 feed.subscription = "denied"; |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
259 return "Ok then :("; |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
260 elseif query["hub.mode"] == feed.subscription then |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
261 module:log("debug", "Confirming %s request to %s", feed.subscription, feed.url) |
2cd667d8edbd
mod_pubsub_feeds: Handle denied subscription requests
Kim Alvefur <zash@zash.se>
parents:
1455
diff
changeset
|
262 else |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
263 module:log("debug", "Invalid mode: %s", tostring(query["hub.mode"])) |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
264 return 400 |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
265 end |
1454
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
266 local lease_seconds = tonumber(query["hub.lease_seconds"]); |
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
267 if lease_seconds then |
480c6f0576b1
mod_pubsub_feeds: Update to 0.4 version of PubSubHubbub
Kim Alvefur <zash@zash.se>
parents:
1453
diff
changeset
|
268 feed.lease_expires = time() + lease_seconds - refresh_interval * 2; |
323
433bf7dc3e7a
mod_pubsub_feed: Stricter verification handling. Correctly echo the hubs challenge.
Kim Alvefur <zash@zash.se>
parents:
322
diff
changeset
|
269 end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
270 return query["hub.challenge"]; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
271 end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
272 return 400; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
273 elseif method == "POST" then |
718
a37e4149ccd1
mod_pubsub_feeds: Prepare for making it possible to unsubscribe, and some other minor changes.
Kim Alvefur <zash@zash.se>
parents:
717
diff
changeset
|
274 if #body > 0 then |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
275 module:log("debug", "got %d bytes PuSHed for %s", #body, query.node); |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
276 local signature = request.headers.x_hub_signature; |
325
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
277 if feed.secret then |
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
278 local localsig = "sha1=" .. hmac_sha1(feed.secret, body, true); |
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
279 if localsig ~= signature then |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
280 module:log("debug", "Invalid signature, got %s but wanted %s", tostring(signature), tostring(localsig)); |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
281 return 401; |
325
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
282 end |
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
283 module:log("debug", "Valid signature"); |
4e50e591a7fc
mod_pubsub_feed: Implement signature verification
Kim Alvefur <zash@zash.se>
parents:
324
diff
changeset
|
284 end |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
285 feed.data = body; |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
286 update_entry(feed); |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
287 return 202; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
288 end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
289 return 400; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
290 end |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
291 return 501; |
322
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
292 end |
637dc0a04052
mod_pubsub_feed: Implement PubSubHubbub subscriber
Kim Alvefur <zash@zash.se>
parents:
300
diff
changeset
|
293 |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
294 if use_pubsubhubub then |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
295 module:provides("http", { |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
296 default_path = "/callback"; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
297 route = { |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
298 GET = handle_http_request; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
299 POST = handle_http_request; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
300 -- This all? |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
301 }; |
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
302 }); |
278
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
303 end |
653c1826739e
mod_pubsub_feed: Fetches Atom feeds and publishes to PubSub
Kim Alvefur <zash@zash.se>
parents:
diff
changeset
|
304 |
668
343b115ebbea
mod_pubsub_feed: Cleanup and update to new APIs in 0.9
Kim Alvefur <zash@zash.se>
parents:
633
diff
changeset
|
305 module:add_timer(1, refresh_feeds); |