evgeny/modoboa-installer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP: Improved backup/restore system.

Browse Source
This commit is contained in:
Antoine Nguyen
2022-11-06 10:30:24 +01:00
parent 61838dbe4d
commit 2b5edae5d5
12 changed files with 304 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
modoboa_installer/scripts/__init__.py
View File

@@ -6,6 +6,17 @@ import sys
from .. import utils
def load_app_script(appname):
"""Load module corresponding to the given appname."""
try:
script = importlib.import_module(
"modoboa_installer.scripts.{}".format(appname))
except ImportError:
print("Unknown application {}".format(appname))
sys.exit(1)
return script
def install(appname, config, upgrade, restore):
"""Install an application."""
if (config.has_option(appname, "enabled") and
@@ -13,12 +24,7 @@ def install(appname, config, upgrade, restore):
return
utils.printcolor("Installing {}".format(appname), utils.MAGENTA)
try:
script = importlib.import_module(
"modoboa_installer.scripts.{}".format(appname))
except ImportError:
print("Unknown application {}".format(appname))
sys.exit(1)
script = load_app_script(appname)
try:
getattr(script, appname.capitalize())(config, upgrade, restore).run()
except utils.FatalError as inst:
@@ -26,13 +32,16 @@ def install(appname, config, upgrade, restore):
sys.exit(1)
def backup(config, silent_backup, backup_path, nomail):
"""Backup instance"""
script = importlib.import_module(
"modoboa_installer.scripts.backup")
def backup(appname, config, path):
"""Backup an application."""
if (config.has_option(appname, "enabled") and
not config.getboolean(appname, "enabled")):
return
utils.printcolor("Backing up {}".format(appname), utils.MAGENTA)
script = load_app_script(appname)
try:
getattr(script, "Backup")(
config, silent_backup, backup_path, nomail).run()
getattr(script, appname.capitalize())(config, False, False).backup(path)
except utils.FatalError as inst:
utils.printcolor(u"{}".format(inst), utils.RED)
sys.exit(1)

35
modoboa_installer/scripts/amavis.py
View File

@@ -6,7 +6,7 @@ from .. import package
from .. import utils
from . import base
from . import install
from . import backup, install
class Amavis(base.Installer):
@@ -42,14 +42,6 @@ class Amavis(base.Installer):
def get_config_files(self):
"""Return appropriate config files."""
if package.backend.FORMAT == "deb":
if self.restore is not None:
amavis_custom_configuration = os.path.join(
self.restore, "custom/99-custom")
if os.path.isfile(amavis_custom_configuration):
utils.copy_file(amavis_custom_configuration, os.path.join(
self.config_dir, "conf.d"))
utils.printcolor(
"Custom amavis configuration restored.", utils.GREEN)
return [
"conf.d/05-node_id", "conf.d/15-content_filter_mode",
"conf.d/50-user"]
@@ -77,11 +69,6 @@ class Amavis(base.Installer):
def get_sql_schema_path(self):
"""Return schema path."""
if self.restore:
db_dump_path = self._restore_database_dump("amavis")
if db_dump_path is not None:
return db_dump_path
version = package.backend.get_installed_version("amavisd-new")
if version is None:
# Fallback to amavis...
@@ -107,3 +94,23 @@ class Amavis(base.Installer):
"""Additional tasks."""
install("spamassassin", self.config, self.upgrade, self.restore)
install("clamav", self.config, self.upgrade, self.restore)
def custom_backup(self, path):
"""Backup custom configuration if any."""
if package.backend.FORMAT == "deb":
amavis_custom = f"{self.config_dir}/conf.d/99-custom"
if os.path.isfile(amavis_custom):
utils.copy_file(amavis_custom, path)
utils.success("Amavis custom configuration saved!")
backup("spamassassin", path)
def restore(self):
"""Restore custom config files."""
if package.backend.FORMAT != "deb":
return
amavis_custom_configuration = os.path.join(
self.restore, "custom/99-custom")
if os.path.isfile(amavis_custom_configuration):
utils.copy_file(amavis_custom_configuration, os.path.join(
self.config_dir, "conf.d"))
utils.success("Custom amavis configuration restored.")

48
modoboa_installer/scripts/base.py
View File

@@ -54,6 +54,19 @@ class Installer(object):
"""Return a schema to install."""
return None
def get_sql_schema_from_backup(self):
"""Retrieve a dump path from a previous backup."""
utils.printcolor(
f"Trying to restore {self.appname} database from backup.",
utils.MAGENTA
)
database_backup_path = os.path.join(
self.restore, f"databases/{self.appname}.sql")
if os.path.isfile(database_backup_path):
utils.success(f"SQL dump found in backup for {self.appname}!")
return database_backup_path
return None
def get_file_path(self, fname):
"""Return the absolute path of this file."""
return os.path.abspath(
@@ -67,6 +80,10 @@ class Installer(object):
return
self.backend.create_user(self.dbuser, self.dbpasswd)
self.backend.create_database(self.dbname, self.dbuser)
schema = None
if self.restore:
schema = self.get_sql_schema_from_backup()
if not schema:
schema = self.get_sql_schema_path()
if schema:
self.backend.load_sql_file(
@@ -138,6 +155,16 @@ class Installer(object):
dst = os.path.join(self.config_dir, dst)
utils.copy_from_template(src, dst, context)
def backup(self, path):
if self.with_db:
self._dump_database(path)
custom_backup_path = os.path.join(path, "custom")
self.custom_backup(custom_backup_path)
def restore(self):
"""Restore from a previous backup."""
pass
def get_daemon_name(self):
"""Return daemon name if defined."""
return self.daemon_name if self.daemon_name else self.appname
@@ -157,22 +184,17 @@ class Installer(object):
if not self.upgrade:
self.setup_database()
self.install_config_files()
if self.restore:
self.restore()
self.post_run()
self.restart_daemon()
def _restore_database_dump(self, app_name):
"""Restore database dump from a dump."""
utils.printcolor(
f"Trying to restore {app_name} database from backup.", utils.MAGENTA)
database_backup_path = os.path.join(
self.restore, f"databases/{app_name}.sql")
if os.path.isfile(database_backup_path):
utils.printcolor(
f"{app_name.capitalize()} database backup found ! Restoring...", utils.GREEN)
return database_backup_path
utils.printcolor(
f"{app_name.capitalize()} database backup not found, creating empty database.", utils.RED)
def _dump_database(self, backup_path: str):
"""Create a new database dump for this app."""
target_dir = os.path.join(backup_path, "databases")
target_file = os.path.join(target_dir, f"{self.appname}.sql")
self.backend.dump_database(
self.dbname, self.dbuser, self.dbpasswd, target_file)
def pre_run(self):
"""Tasks to execute before the installer starts."""

56
modoboa_installer/scripts/dovecot.py
View File

@@ -96,28 +96,6 @@ class Dovecot(base.Installer):
def post_run(self):
"""Additional tasks."""
if self.restore is not None:
mail_dir = os.path.join(self.restore, "mails/")
if len(os.listdir(mail_dir)) > 0:
utils.printcolor(
"Copying mail backup over dovecot directory.", utils.GREEN)
if os.path.exists(self.home_dir):
shutil.rmtree(self.home_dir)
shutil.copytree(mail_dir, self.home_dir)
# Resetting permission for vmail
for dirpath, dirnames, filenames in os.walk(self.home_dir):
shutil.chown(dirpath, self.user, self.user)
for filename in filenames:
shutil.chown(os.path.join(dirpath, filename),
self.user, self.user)
else:
utils.printcolor(
"It seems that mails were not backed up, skipping mail restoration.",
utils.MAGENTA
)
if self.dbengine == "postgres":
dbname = self.config.get("modoboa", "dbname")
dbuser = self.config.get("modoboa", "dbuser")
@@ -156,3 +134,37 @@ class Dovecot(base.Installer):
"service {} {} > /dev/null 2>&1".format(self.appname, action),
capture_output=False)
system.enable_service(self.get_daemon_name())
def backup(self, path):
"""Backup emails."""
utils.printcolor("Backing up mails", utils.MAGENTA)
if not os.path.exists(self.home_dir) or os.path.isfile(self.home_dir):
utils.printcolor("Error backing up emails, provided path "
f" ({self.home_dir}) seems not right...", utils.RED)
return
dst = os.path.join(path, "mails/")
if os.path.exists(dst):
shutil.rmtree(dst)
shutil.copytree(self.home_dir, dst)
utils.printcolor("Mail backup complete!", utils.GREEN)
def restore(self):
"""Restore emails."""
mail_dir = os.path.join(self.restore, "mails/")
if len(os.listdir(mail_dir)) > 0:
utils.success("Copying mail backup over dovecot directory.")
if os.path.exists(self.home_dir):
shutil.rmtree(self.home_dir)
shutil.copytree(mail_dir, self.home_dir)
# Resetting permission for vmail
for dirpath, dirnames, filenames in os.walk(self.home_dir):
shutil.chown(dirpath, self.user, self.user)
for filename in filenames:
shutil.chown(os.path.join(dirpath, filename),
self.user, self.user)
else:
utils.printcolor(
"It seems that emails were not backed up, skipping restoration.",
utils.MAGENTA
)

8
modoboa_installer/scripts/modoboa.py
View File

@@ -192,14 +192,6 @@ class Modoboa(base.Installer):
self.backend.grant_access(
self.config.get("amavis", "dbname"), self.dbuser)
def get_sql_schema_path(self):
if self.restore is not None:
db_dump_path = self._restore_database_dump("modoboa")
if db_dump_path is not None:
return db_dump_path
return super(Modoboa, self).get_sql_schema_path()
def get_packages(self):
"""Include extra packages if needed."""
packages = super(Modoboa, self).get_packages()

10
modoboa_installer/scripts/opendkim.py
View File

@@ -2,6 +2,7 @@
import os
import pwd
import shutil
import stat
from .. import database
@@ -121,3 +122,12 @@ class Opendkim(base.Installer):
"s/^After=(.*)$/After=$1 {}/".format(dbservice))
utils.exec_cmd(
"perl -pi -e '{}' /lib/systemd/system/opendkim.service".format(pattern))
def custom_backup(self, path):
"""Backup DKIM keys."""
storage_dir = self.config.get(
"opendkim", "keys_storage_dir", fallback="/var/lib/dkim")
if os.path.isdir(storage_dir):
shutil.copytree(storage_dir, os.path.join(path, "dkim"))
utils.printcolor(
"DKIM keys saved!", utils.GREEN)

6
modoboa_installer/scripts/postfix.py
View File

@@ -10,7 +10,7 @@ from .. import package
from .. import utils
from . import base
from . import install
from . import backup, install
class Postfix(base.Installer):
@@ -98,3 +98,7 @@ class Postfix(base.Installer):
# Postwhite
install("postwhite", self.config, self.upgrade, self.restore)
def backup(self, path):
"""Launch postwhite backup."""
backup("postwhite", path)

25
modoboa_installer/scripts/postwhite.py
View File

@@ -45,17 +45,26 @@ class Postwhite(base.Installer):
"""Additionnal tasks."""
install_dir = "/usr/local/bin"
self.install_from_archive(SPF_TOOLS_REPOSITORY, install_dir)
postw_dir = self.install_from_archive(
self.postw_dir = self.install_from_archive(
POSTWHITE_REPOSITORY, install_dir)
# Attempt to restore config file from backup
if self.restore is not None:
postw_bin = os.path.join(self.postw_dir, "postwhite")
utils.exec_cmd("{} /etc/postwhite.conf".format(postw_bin))
def custom_backup(self, path):
"""Backup custom configuration if any."""
postswhite_custom = "/etc/postwhite.conf"
if os.path.isfile(postswhite_custom):
utils.copy_file(postswhite_custom, path)
utils.printcolor(
"Postwhite configuration saved!", utils.GREEN)
def restore(self):
"""Restore config files."""
postwhite_backup_configuration = os.path.join(
self.restore, "custom/postwhite.conf")
if os.path.isfile(postwhite_backup_configuration):
utils.copy_file(postwhite_backup_configuration, self.config_dir)
utils.printcolor(
"postwhite.conf restored from backup", utils.GREEN)
utils.success("postwhite.conf restored from backup")
else:
utils.copy_file(os.path.join(postw_dir, "postwhite.conf"), self.config_dir)
postw_bin = os.path.join(postw_dir, "postwhite")
utils.exec_cmd("{} /etc/postwhite.conf".format(postw_bin))
utils.copy_file(
os.path.join(self.postw_dir, "postwhite.conf"), self.config_dir)

19
modoboa_installer/scripts/radicale.py
View File

@@ -71,8 +71,10 @@ class Radicale(base.Installer):
stat.S_IROTH | stat.S_IXOTH,
0, 0
)
# Attempt to restore radicale collections from backup
if self.restore is not None:
super().install_config_files()
def restore(self):
"""Restore collections."""
radicale_backup = os.path.join(
self.restore, "custom/radicale")
if os.path.isdir(radicale_backup):
@@ -80,9 +82,7 @@ class Radicale(base.Installer):
if os.path.isdir(restore_target):
shutil.rmtree(restore_target)
shutil.copytree(radicale_backup, restore_target)
utils.printcolor(
"Radicale collections restored from backup", utils.GREEN)
super(Radicale, self).install_config_files()
utils.success("Radicale collections restored from backup")
def post_run(self):
"""Additional tasks."""
@@ -93,3 +93,12 @@ class Radicale(base.Installer):
system.enable_service(daemon_name)
utils.exec_cmd("service {} stop".format(daemon_name))
utils.exec_cmd("service {} start".format(daemon_name))
def custom_backup(self, path):
"""Backup collections."""
radicale_backup = os.path.join(self.config.get(
"radicale", "home_dir", fallback="/srv/radicale"), "collections")
if os.path.isdir(radicale_backup):
shutil.copytree(radicale_backup, os.path.join(
path, "radicale"))
utils.printcolor("Radicale files saved", utils.GREEN)

4
modoboa_installer/scripts/spamassassin.py
View File

@@ -25,10 +25,6 @@ class Spamassassin(base.Installer):
def get_sql_schema_path(self):
"""Return SQL schema."""
if self.restore is not None:
db_dump_path = self._restore_database_dump("spamassassin")
if db_dump_path is not None:
return db_dump_path
if self.dbengine == "postgres":
fname = "bayes_pg.sql"
else:

61
modoboa_installer/utils.py
View File

@@ -226,6 +226,16 @@ def printcolor(message, color):
print(message)
def error(message):
"""Print error message."""
printcolor(message, RED)
def success(message):
"""Print success message."""
printcolor(message, GREEN)
def convert_version_to_int(version):
"""Convert a version string to an integer."""
number_bits = (8, 8, 16)
@@ -338,3 +348,54 @@ def gen_config(dest, interactive=False):
os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR)
def validate_backup_path(path: str, silent_mode: bool):
"""Check if provided backup path is valid or not."""
path_exists = os.path.exists(path)
if path_exists and os.path.isfile(path):
printcolor(
"Error, you provided a file instead of a directory!", RED)
return None
if not path_exists:
if not silent_mode:
create_dir = input(
f"\"{path}\" doesn't exist, would you like to create it? [Y/n]\n"
).lower()
if silent_mode or (not silent_mode and create_dir.startswith("y")):
pw = pwd.getpwnam("root")
mkdir_safe(path, stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3])
else:
printcolor(
"Error, backup directory not present.", RED
)
return None
if len(os.listdir(path)) != 0:
if not silent_mode:
delete_dir = input(
"Warning: backup directory is not empty, it will be purged if you continue... [Y/n]\n").lower()
if silent_mode or (not silent_mode and delete_dir.startswith("y")):
try:
os.remove(os.path.join(path, "installer.cfg"))
except FileNotFoundError:
pass
shutil.rmtree(os.path.join(path, "custom"),
ignore_errors=False)
shutil.rmtree(os.path.join(path, "mails"), ignore_errors=False)
shutil.rmtree(os.path.join(path, "databases"),
ignore_errors=False)
else:
printcolor(
"Error: backup directory not clean.", RED
)
return None
backup_path = path
pw = pwd.getpwnam("root")
for dir in ["custom/", "databases/"]:
mkdir_safe(os.path.join(backup_path, dir),
stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3])
return backup_path

99
run.py
View File

@@ -3,6 +3,7 @@
"""An installer for Modoboa."""
import argparse
import datetime
import os
try:
import configparser
@@ -11,6 +12,7 @@ except ImportError:
import sys
from modoboa_installer import compatibility_matrix
from modoboa_installer import constants
from modoboa_installer import package
from modoboa_installer import scripts
from modoboa_installer import ssl
@@ -18,6 +20,19 @@ from modoboa_installer import system
from modoboa_installer import utils
PRIMARY_APPS = [
"amavis",
"modoboa",
"automx",
"radicale",
"uwsgi",
"nginx",
"opendkim",
"postfix",
"dovecot"
]
def installation_disclaimer(args, config):
"""Display installation disclaimer."""
hostname = config.get("general", "hostname")
@@ -62,6 +77,42 @@ def restore_disclaimer():
utils.BLUE)
def backup_system(config, args):
"""Launch backup procedure."""
backup_disclaimer()
backup_path = None
if args.silent_backup:
if not args.backup_path:
if config.has_option("backup", "default_path"):
path = config.get("backup", "default_path")
else:
path = constants.DEFAULT_BACKUP_DIRECTORY
date = datetime.datetime.now().strftime("%m_%d_%Y_%H_%M")
path = os.path.join(path, f"backup_{date}")
else:
path = args.backup_path
backup_path = utils.validate_backup_path(path)
if not backup_path:
utils.printcolor(f"Path provided: {path}", utils.BLUE)
return
else:
user_value = None
while not user_value or not backup_path:
utils.printcolor(
"Enter backup path (it must be an empty directory)",
utils.MAGENTA
)
utils.printcolor("CTRL+C to cancel", utils.MAGENTA)
user_value = utils.user_input("-> ")
backup_path = utils.validate_backup_path(user_value)
# Backup configuration file
utils.copy_file(args.configfile, backup_path)
# Backup applications
for app in PRIMARY_APPS:
scripts.backup(app, config, backup_path)
def main(input_args):
"""Install process."""
parser = argparse.ArgumentParser()
@@ -98,9 +149,6 @@ def main(input_args):
"--silent-backup", action="store_true", default=False,
help="For script usage, do not require user interaction "
"backup will be saved at ./modoboa_backup/Backup_M_Y_d_H_M if --backup-path is not provided")
parser.add_argument(
"--no-mail-backup", action="store_true", default=False,
help="Disable mail backup (save space)")
parser.add_argument(
"--restore", type=str, metavar="path",
help="Restore a previously backup up modoboa instance on a NEW machine. "
@@ -117,17 +165,19 @@ def main(input_args):
is_restoring = False
if args.restore is not None:
is_restoring = True
args.configfile = os.path.join(args.restore, "installer.cfg")
args.configfile = os.path.join(args.restore, args.configfile)
if not os.path.exists(args.configfile):
utils.printcolor("installer.cfg from backup not found!", utils.RED)
utils.error(
"Installer configuration file not found in backup!"
)
sys.exit(1)
utils.printcolor("Welcome to Modoboa installer!\n", utils.GREEN)
utils.success("Welcome to Modoboa installer!\n")
is_config_file_available = utils.check_config_file(
args.configfile, args.interactive, args.upgrade, args.backup, is_restoring)
if is_config_file_available and args.backup:
utils.printcolor("No config file found,", utils.RED)
utils.error("No config file found,")
return
if args.stop_after_configfile_check:
@@ -142,14 +192,14 @@ def main(input_args):
config.set("dovecot", "domain", args.domain)
config.set("modoboa", "version", args.version)
config.set("modoboa", "install_beta", str(args.beta))
if args.backup or args.silent_backup:
backup_system(config, args)
return
# Display disclaimer python 3 linux distribution
if args.upgrade:
upgrade_disclaimer(config)
elif args.backup or args.silent_backup:
backup_disclaimer()
scripts.backup(config, args.silent_backup,
args.backup_path, args.no_mail_backup)
return
elif args.restore:
restore_disclaimer()
scripts.restore_prep(args.restore)
@@ -175,33 +225,26 @@ def main(input_args):
utils.printcolor(
"The process can be long, feel free to take a coffee "
"and come back later ;)", utils.BLUE)
utils.printcolor("Starting...", utils.GREEN)
utils.success("Starting...")
package.backend.prepare_system()
package.backend.install_many(["sudo", "wget"])
ssl_backend = ssl.get_backend(config)
if ssl_backend and not args.upgrade:
ssl_backend.generate_cert()
scripts.install("amavis", config, args.upgrade, args.restore)
scripts.install("modoboa", config, args.upgrade, args.restore)
scripts.install("automx", config, args.upgrade, args.restore)
scripts.install("radicale", config, args.upgrade, args.restore)
scripts.install("uwsgi", config, args.upgrade, args.restore)
scripts.install("nginx", config, args.upgrade, args.restore)
scripts.install("opendkim", config, args.upgrade, args.restore)
scripts.install("postfix", config, args.upgrade, args.restore)
scripts.install("dovecot", config, args.upgrade, args.restore)
for appname in PRIMARY_APPS:
scripts.install(appname, config, args.upgrade, args.restore)
system.restart_service("cron")
package.backend.restore_system()
if not args.restore:
utils.printcolor(
utils.success(
"Congratulations! You can enjoy Modoboa at https://{} (admin:password)"
.format(config.get("general", "hostname")),
utils.GREEN)
.format(config.get("general", "hostname"))
)
else:
utils.printcolor(
utils.success(
"Restore complete! You can enjoy Modoboa at https://{} (same credentials as before)"
.format(config.get("general", "hostname")),
utils.GREEN)
.format(config.get("general", "hostname"))
)
if __name__ == "__main__":