prosody
4d702f0c6273
tools/migration/prosody-migrator.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Comparison

tools/migration/prosody-migrator.lua @ 10003:4d702f0c6273

migrator: Rewrite to use storage modules This allows migrating to and from any storage module that supports the right methods. Based on experimental mod_migrate work.
author Kim Alvefur <zash@zash.se>
date Sun, 05 May 2019 21:32:34 +0200
parent 8062:739bb455cafd
child 10004:e057e8318130
comparison
equal deleted inserted replaced
10002:b6b5b9d7417d 10003:4d702f0c6273
1 #!/usr/bin/env lua 1 #!/usr/bin/env lua
2 2
3 CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR"); 3 CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR");
4 CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR"); 4 CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR");
5 CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR");
6 CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR");
5 7
6 -- Substitute ~ with path to home directory in paths 8 -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
7 if CFG_CONFIGDIR then 9
8 CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME")); 10 local function is_relative(path)
11 local path_sep = package.config:sub(1,1);
12 return ((path_sep == "/" and path:sub(1,1) ~= "/")
13 or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\")))
9 end 14 end
10 15
16 -- Tell Lua where to find our libraries
11 if CFG_SOURCEDIR then 17 if CFG_SOURCEDIR then
12 CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME")); 18 local function filter_relative_paths(path)
19 if is_relative(path) then return ""; end
20 end
21 local function sanitise_paths(paths)
22 return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";"));
23 end
24 package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path);
25 package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath);
26 end
27
28 -- Substitute ~ with path to home directory in data path
29 if CFG_DATADIR then
30 if os.getenv("HOME") then
31 CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME"));
32 end
13 end 33 end
14 34
15 local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua"; 35 local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua";
36
37 local startup = require "util.startup";
38 startup.prosodyctl();
39 -- TODO startup.migrator ?
16 40
17 -- Command-line parsing 41 -- Command-line parsing
18 local options = {}; 42 local options = {};
19 local i = 1; 43 local i = 1;
20 while arg[i] do 44 while arg[i] do
27 else 51 else
28 i = i + 1; 52 i = i + 1;
29 end 53 end
30 end 54 end
31 55
32 if CFG_SOURCEDIR then
33 package.path = CFG_SOURCEDIR.."/?.lua;"..package.path;
34 package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath;
35 else
36 package.path = "../../?.lua;"..package.path
37 package.cpath = "../../?.so;"..package.cpath
38 end
39 56
40 local envloadfile = require "util.envload".envloadfile; 57 local envloadfile = require "util.envload".envloadfile;
41 58
42 local config_file = options.config or default_config; 59 local config_file = options.config or default_config;
43 local from_store = arg[1] or "input"; 60 local from_store = arg[1] or "input";
67 if not config[to_store] then 84 if not config[to_store] then
68 have_err = true; 85 have_err = true;
69 print("Error: Output store '"..to_store.."' not found in the config file."); 86 print("Error: Output store '"..to_store.."' not found in the config file.");
70 end 87 end
71 88
72 function load_store_handler(name) 89 for store, conf in pairs(config) do -- COMPAT
73 local store_type = config[name].type; 90 if conf.type == "prosody_files" then
74 if not store_type then 91 conf.type = "internal";
75 print("Error: "..name.." store type not specified in the config file"); 92 elseif conf.type == "prosody_sql" then
76 return false; 93 conf.type = "sql";
77 else
78 local ok, err = pcall(require, "migrator."..store_type);
79 if not ok then
80 print(("Error: Failed to initialize '%s' store:\n\t%s")
81 :format(name, err));
82 return false;
83 end
84 end 94 end
85 return true;
86 end 95 end
87
88 have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output"));
89 96
90 if have_err then 97 if have_err then
91 print(""); 98 print("");
92 print("Usage: "..arg[0].." FROM_STORE TO_STORE"); 99 print("Usage: "..arg[0].." FROM_STORE TO_STORE");
93 print("If no stores are specified, 'input' and 'output' are used."); 100 print("If no stores are specified, 'input' and 'output' are used.");
99 end 106 end
100 print(""); 107 print("");
101 os.exit(1); 108 os.exit(1);
102 end 109 end
103 110
104 local itype = config[from_store].type; 111 local async = require "util.async";
105 local otype = config[to_store].type; 112 local server = require "net.server";
106 local reader = require("migrator."..itype).reader(config[from_store]); 113 local watchers = {
107 local writer = require("migrator."..otype).writer(config[to_store]); 114 error = function (_, err)
115 error(err);
116 end;
117 waiting = function ()
118 server.loop();
119 end;
120 };
108 121
109 local json = require "util.json"; 122 local cm = require "core.configmanager";
123 local hm = require "core.hostmanager";
124 local sm = require "core.storagemanager";
125 local um = require "core.usermanager";
126
127 local function users(store, host)
128 if store.users then
129 return store:users();
130 else
131 return um.users(host);
132 end
133 end
134
135 local function prepare_config(host, conf)
136 if conf.type == "internal" then
137 sm.olddm.set_data_path(conf.path or prosody.paths.data);
138 elseif conf.type == "sql" then
139 cm.set(host, "sql", conf);
140 end
141 end
142
143 local function get_driver(host, conf)
144 prepare_config(host, conf);
145 return assert(sm.load_driver(host, conf.type));
146 end
147
148 local migration_runner = async.runner(function (job)
149 for host, stores in pairs(job.input.hosts) do
150 prosody.hosts[host] = startup.make_host(host);
151 sm.initialize_host(host);
152 um.initialize_host(host);
153
154 local input_driver = get_driver(host, job.input);
155
156 local output_driver = get_driver(host, job.output);
157
158 for _, store in ipairs(stores) do
159 local p, typ = store:match("()%-(%w+)$");
160 if typ then store = store:sub(1, p-1); else typ = "keyval"; end
161 log("info", "Migrating host %s store %s (%s)", host, store, typ);
162
163 local origin = assert(input_driver:open(store, typ));
164 local destination = assert(output_driver:open(store, typ));
165
166 if typ == "keyval" then -- host data
167 local data, err = origin:get(nil);
168 assert(not err, err);
169 assert(destination:set(nil, data));
170 end
171
172 for user in users(origin, host) do
173 if typ == "keyval" then
174 local data, err = origin:get(user);
175 assert(not err, err);
176 assert(destination:set(user, data));
177 else
178 error("Don't know how to migrate data of type '"..typ.."'.");
179 end
180 end
181 end
182 end
183 end, watchers);
110 184
111 io.stderr:write("Migrating...\n"); 185 io.stderr:write("Migrating...\n");
112 for x in reader do 186
113 --print(json.encode(x)) 187 migration_runner:run({ input = config[from_store], output = config[to_store] });
114 writer(x); 188
115 end
116 writer(nil); -- close
117 io.stderr:write("Done!\n"); 189 io.stderr:write("Done!\n");