Annotate

tools/migration/migrator/prosody_files.lua @ 4216:ff80a8471e86

tools/migration/*: Numerous changes and restructuring, and the addition of a Makefile
author Matthew Wild <mwild1@gmail.com>
date Sat, 26 Feb 2011 00:23:48 +0000
parent 4197:tools/migration/prosody_files.lua@bef732980436
child 4239:69fe5fd861e7
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
4197
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
1
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
2 local print = print;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
3 local assert = assert;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
4 local setmetatable = setmetatable;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
5 local tonumber = tonumber;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
6 local char = string.char;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
7 local coroutine = coroutine;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
8 local lfs = require "lfs";
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
9 local loadfile = loadfile;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
10 local setfenv = setfenv;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
11 local pcall = pcall;
4216
ff80a8471e86 tools/migration/*: Numerous changes and restructuring, and the addition of a Makefile
Matthew Wild <mwild1@gmail.com>
parents: 4197
diff changeset
12 local mtools = require "migrator.mtools";
4197
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
13 local next = next;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
14 local pairs = pairs;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
15 local json = require "util.json";
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
16
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
17 prosody = {};
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
18 local dm = require "util.datamanager"
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
19
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
20 module "prosody_files"
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
21
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
22 local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
23 local function is_file(path) return lfs.attributes(path, "mode") == "file"; end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
24 local function clean_path(path)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
25 return path:gsub("\\", "/"):gsub("//+", "/");
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
26 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
27 local encode, decode; do
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
28 local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end });
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
29 decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
30 encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
31 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
32 local function decode_dir(x)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
33 if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
34 return decode(x);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
35 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
36 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
37 local function decode_file(x)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
38 if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
39 return decode(x:gsub("%.dat$", ""));
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
40 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
41 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
42 local function prosody_dir(path, ondir, onfile, ...)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
43 for x in lfs.dir(path) do
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
44 local xpath = path.."/"..x;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
45 if decode_dir(x) and is_dir(xpath) then
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
46 ondir(xpath, x, ...);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
47 elseif decode_file(x) and is_file(xpath) then
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
48 onfile(xpath, x, ...);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
49 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
50 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
51 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
52
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
53 local function handle_root_file(path, name)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
54 --print("root file: ", decode_file(name))
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
55 coroutine.yield { user = nil, host = nil, store = decode_file(name) };
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
56 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
57 local function handle_host_file(path, name, host)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
58 --print("host file: ", decode_dir(host).."/"..decode_file(name))
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
59 coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) };
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
60 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
61 local function handle_store_file(path, name, store, host)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
62 --print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store))
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
63 coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) };
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
64 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
65 local function handle_host_store(path, name, host)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
66 prosody_dir(path, function() end, handle_store_file, name, host);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
67 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
68 local function handle_host_dir(path, name)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
69 prosody_dir(path, handle_host_store, handle_host_file, name);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
70 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
71 local function handle_root_dir(path)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
72 prosody_dir(path, handle_host_dir, handle_root_file);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
73 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
74
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
75 local function decode_user(item)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
76 local userdata = {
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
77 user = item[1].user;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
78 host = item[1].host;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
79 stores = {};
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
80 };
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
81 for i=1,#item do -- loop over stores
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
82 local result = {};
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
83 local store = item[i];
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
84 userdata.stores[store.store] = store.data;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
85 store.user = nil; store.host = nil; store.store = nil;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
86 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
87 return userdata;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
88 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
89
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
90 function reader(input)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
91 local path = clean_path(assert(input.path, "no input.path specified"));
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
92 assert(is_dir(path), "input.path is not a directory");
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
93 local iter = coroutine.wrap(function()handle_root_dir(path);end);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
94 -- get per-user stores, sorted
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
95 local iter = mtools.sorted {
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
96 reader = function()
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
97 local x = iter();
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
98 if x then
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
99 dm.set_data_path(path);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
100 x.data = assert(dm.load(x.user, x.host, x.store));
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
101 return x;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
102 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
103 end;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
104 sorter = function(a, b)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
105 local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
106 local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
107 return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
108 end;
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
109 };
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
110 -- merge stores to get users
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
111 iter = mtools.merged(iter, function(a, b)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
112 return (a.host == b.host and a.user == b.user);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
113 end);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
114
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
115 return function()
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
116 local x = iter();
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
117 return x and decode_user(x);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
118 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
119 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
120
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
121 function writer(output)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
122 local path = clean_path(assert(output.path, "no output.path specified"));
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
123 assert(is_dir(path), "output.path is not a directory");
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
124 return function(item)
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
125 if not item then return; end -- end of input
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
126 dm.set_data_path(path);
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
127 for store, data in pairs(item.stores) do
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
128 assert(dm.store(item.user, item.host, store, data));
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
129 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
130 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
131 end
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
132
bef732980436 tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4193
diff changeset
133 return _M;