prosody
tip
tools/migration/prosody-migrator.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Annotate

tools/migration/prosody-migrator.lua @ 13270:14bbfb2cc8dd default tip

lint: Teach luacheck about module:once Silence warning for using this introduced in 9c62ffbdf2ae
author Kim Alvefur <zash@zash.se>
date Sun, 15 Oct 2023 16:43:14 +0200
parent 13142:879a6a33c21b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
4216
ff80a8471e86 tools/migration/*: Numerous changes and restructuring, and the addition of a Makefile
Matthew Wild <mwild1@gmail.com>
parents: 4211
diff changeset
1 #!/usr/bin/env lua
ff80a8471e86 tools/migration/*: Numerous changes and restructuring, and the addition of a Makefile
Matthew Wild <mwild1@gmail.com>
parents: 4211
diff changeset
2
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
3 CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR");
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
4 CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR");
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
5 CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR");
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
6 CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR");
4216
ff80a8471e86 tools/migration/*: Numerous changes and restructuring, and the addition of a Makefile
Matthew Wild <mwild1@gmail.com>
parents: 4211
diff changeset
7
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
8 -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
9
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
10 local function is_relative(path)
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
11 local path_sep = package.config:sub(1,1);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
12 return ((path_sep == "/" and path:sub(1,1) ~= "/")
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
13 or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\")))
4239
69fe5fd861e7 tools/migration: Support for ~/ in paths
Matthew Wild <mwild1@gmail.com>
parents: 4235
diff changeset
14 end
69fe5fd861e7 tools/migration: Support for ~/ in paths
Matthew Wild <mwild1@gmail.com>
parents: 4235
diff changeset
15
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
16 -- Tell Lua where to find our libraries
4239
69fe5fd861e7 tools/migration: Support for ~/ in paths
Matthew Wild <mwild1@gmail.com>
parents: 4235
diff changeset
17 if CFG_SOURCEDIR then
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
18 local function filter_relative_paths(path)
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
19 if is_relative(path) then return ""; end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
20 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
21 local function sanitise_paths(paths)
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
22 return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";"));
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
23 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
24 package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
25 package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
26 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
27
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
28 -- Substitute ~ with path to home directory in data path
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
29 if CFG_DATADIR then
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
30 if os.getenv("HOME") then
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
31 CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME"));
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
32 end
4239
69fe5fd861e7 tools/migration: Support for ~/ in paths
Matthew Wild <mwild1@gmail.com>
parents: 4235
diff changeset
33 end
69fe5fd861e7 tools/migration: Support for ~/ in paths
Matthew Wild <mwild1@gmail.com>
parents: 4235
diff changeset
34
4216
ff80a8471e86 tools/migration/*: Numerous changes and restructuring, and the addition of a Makefile
Matthew Wild <mwild1@gmail.com>
parents: 4211
diff changeset
35 local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua";
4211
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
36
12162
b7ee14ba09c9 migrator: Reuse earlier usage text
Kim Alvefur <zash@zash.se>
parents: 12161
diff changeset
37 local function usage()
12163
59b65cc6312f migrator: Include --options in usage info
Kim Alvefur <zash@zash.se>
parents: 12162
diff changeset
38 print("Usage: " .. arg[0] .. " [OPTIONS] FROM_STORE TO_STORE");
59b65cc6312f migrator: Include --options in usage info
Kim Alvefur <zash@zash.se>
parents: 12162
diff changeset
39 print(" --config FILE Specify config file")
12166
39483b4099b4 migrator: Add option to keep going despite errors
Kim Alvefur <zash@zash.se>
parents: 12165
diff changeset
40 print(" --keep-going Keep going in case of errors");
12387
05c250fa335a Spelling: Fix various spelling mistakes (thanks timeless)
Kim Alvefur <zash@zash.se>
parents: 12168
diff changeset
41 print(" -v, --verbose Increase log-level");
12163
59b65cc6312f migrator: Include --options in usage info
Kim Alvefur <zash@zash.se>
parents: 12162
diff changeset
42 print("");
12162
b7ee14ba09c9 migrator: Reuse earlier usage text
Kim Alvefur <zash@zash.se>
parents: 12161
diff changeset
43 print("If no stores are specified, 'input' and 'output' are used.");
b7ee14ba09c9 migrator: Reuse earlier usage text
Kim Alvefur <zash@zash.se>
parents: 12161
diff changeset
44 end
b7ee14ba09c9 migrator: Reuse earlier usage text
Kim Alvefur <zash@zash.se>
parents: 12161
diff changeset
45
13142
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
46 if not pcall(require, "prosody.loader") then
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
47 pcall(require, "loader");
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
48 end
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
49
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
50 local startup = require "prosody.util.startup";
11728
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
51 do
12161
6e7678f6fe9a migrator: Customise cli argument parsing (--help, --verbose)
Kim Alvefur <zash@zash.se>
parents: 11731
diff changeset
52 startup.parse_args({
6e7678f6fe9a migrator: Customise cli argument parsing (--help, --verbose)
Kim Alvefur <zash@zash.se>
parents: 11731
diff changeset
53 short_params = { v = "verbose", h = "help", ["?"] = "help" };
6e7678f6fe9a migrator: Customise cli argument parsing (--help, --verbose)
Kim Alvefur <zash@zash.se>
parents: 11731
diff changeset
54 value_params = { config = true };
6e7678f6fe9a migrator: Customise cli argument parsing (--help, --verbose)
Kim Alvefur <zash@zash.se>
parents: 11731
diff changeset
55 });
11728
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
56 startup.init_global_state();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
57 prosody.process_type = "migrator";
12161
6e7678f6fe9a migrator: Customise cli argument parsing (--help, --verbose)
Kim Alvefur <zash@zash.se>
parents: 11731
diff changeset
58 if prosody.opts.help then
12162
b7ee14ba09c9 migrator: Reuse earlier usage text
Kim Alvefur <zash@zash.se>
parents: 12161
diff changeset
59 usage();
12161
6e7678f6fe9a migrator: Customise cli argument parsing (--help, --verbose)
Kim Alvefur <zash@zash.se>
parents: 11731
diff changeset
60 os.exit(0);
6e7678f6fe9a migrator: Customise cli argument parsing (--help, --verbose)
Kim Alvefur <zash@zash.se>
parents: 11731
diff changeset
61 end
11728
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
62 startup.force_console_logging();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
63 startup.init_logging();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
64 startup.init_gc();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
65 startup.init_errors();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
66 startup.setup_plugindir();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
67 startup.setup_plugin_install_path();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
68 startup.setup_datadir();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
69 startup.chdir();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
70 startup.read_version();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
71 startup.switch_user();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
72 startup.check_dependencies();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
73 startup.log_startup_warnings();
11729
f37cafeb75d6 migrator: Trick net.server into thinking the config is loaded
Kim Alvefur <zash@zash.se>
parents: 11728
diff changeset
74 prosody.config_loaded = true;
11728
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
75 startup.load_libraries();
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
76 startup.init_http_client();
11731
076ceb405b94 migrator: Silence assert in core.moduleapi
Kim Alvefur <zash@zash.se>
parents: 11730
diff changeset
77 prosody.core_post_stanza = function ()
076ceb405b94 migrator: Silence assert in core.moduleapi
Kim Alvefur <zash@zash.se>
parents: 11730
diff changeset
78 -- silence assert in core.moduleapi
076ceb405b94 migrator: Silence assert in core.moduleapi
Kim Alvefur <zash@zash.se>
parents: 11730
diff changeset
79 error("Attempt to send stanzas from inside migrator.", 0);
076ceb405b94 migrator: Silence assert in core.moduleapi
Kim Alvefur <zash@zash.se>
parents: 11730
diff changeset
80 end
11728
826d57c16d1c migrator: Customise startup sequence to fix #1673 (Thanks acidsys)
Kim Alvefur <zash@zash.se>
parents: 10004
diff changeset
81 end
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
82
4210
4583473dcce4 tools/migration/main.lua: Add command-line parsing, including --config=CONFIG_FILE, and the ability to specify to/from stores to migrate
Matthew Wild <mwild1@gmail.com>
parents: 4166
diff changeset
83 -- Command-line parsing
11730
9bf8a0607d12 migrator: Use parsed command line flags already parsed by util.startup
Kim Alvefur <zash@zash.se>
parents: 11729
diff changeset
84 local options = prosody.opts;
5021
85b2689dbcfe Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents: 4240
diff changeset
85
13142
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
86 local envloadfile = require "prosody.util.envload".envloadfile;
5021
85b2689dbcfe Eliminate direct setfenv usage
Florian Zeitz <florob@babelmonkeys.de>
parents: 4240
diff changeset
87
4211
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
88 local config_file = options.config or default_config;
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
89 local from_store = arg[1] or "input";
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
90 local to_store = arg[2] or "output";
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
91
4166
3ac90743039b tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4162
diff changeset
92 config = {};
3ac90743039b tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4162
diff changeset
93 local config_env = setmetatable({}, { __index = function(t, k) return function(tbl) config[k] = tbl; end; end });
7880
1d998891c967 migrator: Remove wrapper around envloadfile since envloadfile does the right thing in a compatible way
Kim Alvefur <zash@zash.se>
parents: 5776
diff changeset
94 local config_chunk, err = envloadfile(config_file, config_env);
4211
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
95 if not config_chunk then
7895
1e1c18012048 migrator: Fix missing word
Kim Alvefur <zash@zash.se>
parents: 7894
diff changeset
96 print("There was an error loading the config file, check that the file exists");
4211
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
97 print("and that the syntax is correct:");
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
98 print("", err);
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
99 os.exit(1);
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
100 end
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
101
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
102 config_chunk();
4166
3ac90743039b tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4162
diff changeset
103
4211
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
104 local have_err;
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
105 if #arg > 0 and #arg ~= 2 then
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
106 have_err = true;
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
107 print("Error: Incorrect number of parameters supplied.");
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
108 end
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
109 if not config[from_store] then
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
110 have_err = true;
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
111 print("Error: Input store '"..from_store.."' not found in the config file.");
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
112 end
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
113 if not config[to_store] then
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
114 have_err = true;
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
115 print("Error: Output store '"..to_store.."' not found in the config file.");
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
116 end
4235
899ffc1674b5 tools/migration/prosody-migrator.lua: Refactor store handler loading to report errors they throw
Matthew Wild <mwild1@gmail.com>
parents: 4229
diff changeset
117
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
118 for store, conf in pairs(config) do -- COMPAT
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
119 if conf.type == "prosody_files" then
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
120 conf.type = "internal";
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
121 elseif conf.type == "prosody_sql" then
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
122 conf.type = "sql";
4235
899ffc1674b5 tools/migration/prosody-migrator.lua: Refactor store handler loading to report errors they throw
Matthew Wild <mwild1@gmail.com>
parents: 4229
diff changeset
123 end
4211
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
124 end
4235
899ffc1674b5 tools/migration/prosody-migrator.lua: Refactor store handler loading to report errors they throw
Matthew Wild <mwild1@gmail.com>
parents: 4229
diff changeset
125
4211
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
126 if have_err then
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
127 print("");
12162
b7ee14ba09c9 migrator: Reuse earlier usage text
Kim Alvefur <zash@zash.se>
parents: 12161
diff changeset
128 usage();
4211
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
129 print("");
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
130 print("The available stores in your migrator config are:");
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
131 print("");
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
132 for store in pairs(config) do
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
133 print("", store);
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
134 end
4216
ff80a8471e86 tools/migration/*: Numerous changes and restructuring, and the addition of a Makefile
Matthew Wild <mwild1@gmail.com>
parents: 4211
diff changeset
135 print("");
4211
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
136 os.exit(1);
9a12fc2baa37 tools/migration/*.lua: Rename config to migrator.cfg.lua, add error handling for config and command-line parameters
Matthew Wild <mwild1@gmail.com>
parents: 4210
diff changeset
137 end
5776
bd0ff8ae98a8 Remove all trailing whitespace
Florian Zeitz <florob@babelmonkeys.de>
parents: 5021
diff changeset
138
13142
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
139 local async = require "prosody.util.async";
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
140 local server = require "prosody.net.server";
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
141 local watchers = {
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
142 error = function (_, err)
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
143 error(err);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
144 end;
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
145 waiting = function ()
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
146 server.loop();
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
147 end;
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
148 };
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
149
13142
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
150 local cm = require "prosody.core.configmanager";
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
151 local hm = require "prosody.core.hostmanager";
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
152 local sm = require "prosody.core.storagemanager";
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
153 local um = require "prosody.core.usermanager";
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
154
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
155 local function users(store, host)
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
156 if store.users then
12167
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
157 log("debug", "Using store user iterator")
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
158 return store:users();
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
159 else
12387
05c250fa335a Spelling: Fix various spelling mistakes (thanks timeless)
Kim Alvefur <zash@zash.se>
parents: 12168
diff changeset
160 log("debug", "Using usermanager user iterator")
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
161 return um.users(host);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
162 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
163 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
164
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
165 local function prepare_config(host, conf)
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
166 if conf.type == "internal" then
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
167 sm.olddm.set_data_path(conf.path or prosody.paths.data);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
168 elseif conf.type == "sql" then
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
169 cm.set(host, "sql", conf);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
170 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
171 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
172
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
173 local function get_driver(host, conf)
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
174 prepare_config(host, conf);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
175 return assert(sm.load_driver(host, conf.type));
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
176 end
4166
3ac90743039b tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4162
diff changeset
177
12164
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
178 local migrate_once = {
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
179 keyval = function(origin, destination, user)
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
180 local data, err = origin:get(user);
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
181 assert(not err, err);
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
182 assert(destination:set(user, data));
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
183 end;
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
184 archive = function(origin, destination, user)
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
185 local iter, err = origin:find(user);
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
186 assert(iter, err);
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
187 for id, item, when, with in iter do
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
188 assert(destination:append(user, id, item, when, with));
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
189 end
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
190 end;
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
191 }
12168
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
192 migrate_once.pubsub = function(origin, destination, user, prefix, input_driver, output_driver)
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
193 if not user and prefix == "pubsub_" then return end
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
194 local data, err = origin:get(user);
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
195 assert(not err, err);
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
196 if not data then return end
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
197 assert(destination:set(user, data));
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
198 if prefix == "pubsub_" then user = nil end
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
199 for node in pairs(data) do
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
200 local pep_origin = assert(input_driver:open(prefix .. node, "archive"));
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
201 local pep_destination = assert(output_driver:open(prefix .. node, "archive"));
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
202 migrate_once.archive(pep_origin, pep_destination, user);
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
203 end
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
204 end
12164
85f03b29ec72 migrator: Refactor out individual item migrator for code deduplication
Kim Alvefur <zash@zash.se>
parents: 12163
diff changeset
205
12166
39483b4099b4 migrator: Add option to keep going despite errors
Kim Alvefur <zash@zash.se>
parents: 12165
diff changeset
206 if options["keep-going"] then
13142
879a6a33c21b tools: Update imports to use new prosody.* namespace
Kim Alvefur <zash@zash.se>
parents: 12387
diff changeset
207 local xpcall = require "prosody.util.xpcall".xpcall;
12166
39483b4099b4 migrator: Add option to keep going despite errors
Kim Alvefur <zash@zash.se>
parents: 12165
diff changeset
208 for t, f in pairs(migrate_once) do
12168
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
209 migrate_once[t] = function (origin, destination, user, ...)
12167
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
210 local function log_err(err)
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
211 if user then
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
212 log("error", "Error migrating data for user %q: %s", user, err);
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
213 else
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
214 log("error", "Error migrating data for host: %s", err);
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
215 end
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
216 log("debug", "%s", debug.traceback(nil, 2));
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
217 end
12168
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
218 xpcall(f, log_err, origin, destination, user, ...);
12166
39483b4099b4 migrator: Add option to keep going despite errors
Kim Alvefur <zash@zash.se>
parents: 12165
diff changeset
219 end
39483b4099b4 migrator: Add option to keep going despite errors
Kim Alvefur <zash@zash.se>
parents: 12165
diff changeset
220 end
39483b4099b4 migrator: Add option to keep going despite errors
Kim Alvefur <zash@zash.se>
parents: 12165
diff changeset
221 end
39483b4099b4 migrator: Add option to keep going despite errors
Kim Alvefur <zash@zash.se>
parents: 12165
diff changeset
222
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
223 local migration_runner = async.runner(function (job)
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
224 for host, stores in pairs(job.input.hosts) do
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
225 prosody.hosts[host] = startup.make_host(host);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
226 sm.initialize_host(host);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
227 um.initialize_host(host);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
228
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
229 local input_driver = get_driver(host, job.input);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
230
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
231 local output_driver = get_driver(host, job.output);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
232
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
233 for _, store in ipairs(stores) do
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
234 local p, typ = store:match("()%-(%w+)$");
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
235 if typ then store = store:sub(1, p-1); else typ = "keyval"; end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
236 log("info", "Migrating host %s store %s (%s)", host, store, typ);
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
237
12168
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
238 local migrate = assert(migrate_once[typ], "Unknown store type: "..typ);
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
239
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
240 local prefix = store .. "_";
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
241 if typ == "pubsub" then typ = "keyval"; end
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
242 if store == "pubsub_nodes" then prefix = "pubsub_"; end
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
243
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
244 local origin = assert(input_driver:open(store, typ));
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
245 local destination = assert(output_driver:open(store, typ));
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
246
12168
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
247 migrate(origin, destination, nil, prefix, input_driver, output_driver); -- host data
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
248
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
249 for user in users(origin, host) do
12167
03f551510b1e migrator: Enhance logging
Kim Alvefur <zash@zash.se>
parents: 12166
diff changeset
250 log("info", "Migrating user %s@%s store %s (%s)", user, host, store, typ);
12168
33e856c65033 migrator: Support pubsub and pep as a special-case
Kim Alvefur <zash@zash.se>
parents: 12167
diff changeset
251 migrate(origin, destination, user, prefix, input_driver, output_driver);
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
252 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
253 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
254 end
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
255 end, watchers);
4166
3ac90743039b tools/migration/*.lua: Convert to unix line endings
Matthew Wild <mwild1@gmail.com>
parents: 4162
diff changeset
256
4240
b3d9063aad4d tools/migration/prosody-migrator.lua: Add messages to show when migration is in progress
Matthew Wild <mwild1@gmail.com>
parents: 4239
diff changeset
257 io.stderr:write("Migrating...\n");
10003
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
258
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
259 migration_runner:run({ input = config[from_store], output = config[to_store] });
4d702f0c6273 migrator: Rewrite to use storage modules
Kim Alvefur <zash@zash.se>
parents: 8062
diff changeset
260
4240
b3d9063aad4d tools/migration/prosody-migrator.lua: Add messages to show when migration is in progress
Matthew Wild <mwild1@gmail.com>
parents: 4239
diff changeset
261 io.stderr:write("Done!\n");