From 2b5edae5d53ee079295d859e47a0b2626084e001 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 6 Nov 2022 10:30:24 +0100 Subject: [PATCH 1/4] WIP: Improved backup/restore system. --- modoboa_installer/scripts/__init__.py | 33 ++++--- modoboa_installer/scripts/amavis.py | 35 +++++--- modoboa_installer/scripts/base.py | 50 ++++++++--- modoboa_installer/scripts/dovecot.py | 56 +++++++----- modoboa_installer/scripts/modoboa.py | 8 -- modoboa_installer/scripts/opendkim.py | 10 +++ modoboa_installer/scripts/postfix.py | 6 +- modoboa_installer/scripts/postwhite.py | 33 ++++--- modoboa_installer/scripts/radicale.py | 33 ++++--- modoboa_installer/scripts/spamassassin.py | 4 - modoboa_installer/utils.py | 65 +++++++++++++- run.py | 101 +++++++++++++++------- 12 files changed, 304 insertions(+), 130 deletions(-) diff --git a/modoboa_installer/scripts/__init__.py b/modoboa_installer/scripts/__init__.py index 94efdcb..8cf0047 100644 --- a/modoboa_installer/scripts/__init__.py +++ b/modoboa_installer/scripts/__init__.py @@ -6,6 +6,17 @@ import sys from .. import utils +def load_app_script(appname): + """Load module corresponding to the given appname.""" + try: + script = importlib.import_module( + "modoboa_installer.scripts.{}".format(appname)) + except ImportError: + print("Unknown application {}".format(appname)) + sys.exit(1) + return script + + def install(appname, config, upgrade, restore): """Install an application.""" if (config.has_option(appname, "enabled") and @@ -13,12 +24,7 @@ def install(appname, config, upgrade, restore): return utils.printcolor("Installing {}".format(appname), utils.MAGENTA) - try: - script = importlib.import_module( - "modoboa_installer.scripts.{}".format(appname)) - except ImportError: - print("Unknown application {}".format(appname)) - sys.exit(1) + script = load_app_script(appname) try: getattr(script, appname.capitalize())(config, upgrade, restore).run() except utils.FatalError as inst: @@ -26,13 +32,16 @@ def install(appname, config, upgrade, restore): sys.exit(1) -def backup(config, silent_backup, backup_path, nomail): - """Backup instance""" - script = importlib.import_module( - "modoboa_installer.scripts.backup") +def backup(appname, config, path): + """Backup an application.""" + if (config.has_option(appname, "enabled") and + not config.getboolean(appname, "enabled")): + return + + utils.printcolor("Backing up {}".format(appname), utils.MAGENTA) + script = load_app_script(appname) try: - getattr(script, "Backup")( - config, silent_backup, backup_path, nomail).run() + getattr(script, appname.capitalize())(config, False, False).backup(path) except utils.FatalError as inst: utils.printcolor(u"{}".format(inst), utils.RED) sys.exit(1) diff --git a/modoboa_installer/scripts/amavis.py b/modoboa_installer/scripts/amavis.py index 8988695..4208bb0 100644 --- a/modoboa_installer/scripts/amavis.py +++ b/modoboa_installer/scripts/amavis.py @@ -6,7 +6,7 @@ from .. import package from .. import utils from . import base -from . import install +from . import backup, install class Amavis(base.Installer): @@ -42,14 +42,6 @@ class Amavis(base.Installer): def get_config_files(self): """Return appropriate config files.""" if package.backend.FORMAT == "deb": - if self.restore is not None: - amavis_custom_configuration = os.path.join( - self.restore, "custom/99-custom") - if os.path.isfile(amavis_custom_configuration): - utils.copy_file(amavis_custom_configuration, os.path.join( - self.config_dir, "conf.d")) - utils.printcolor( - "Custom amavis configuration restored.", utils.GREEN) return [ "conf.d/05-node_id", "conf.d/15-content_filter_mode", "conf.d/50-user"] @@ -77,11 +69,6 @@ class Amavis(base.Installer): def get_sql_schema_path(self): """Return schema path.""" - if self.restore: - db_dump_path = self._restore_database_dump("amavis") - if db_dump_path is not None: - return db_dump_path - version = package.backend.get_installed_version("amavisd-new") if version is None: # Fallback to amavis... @@ -107,3 +94,23 @@ class Amavis(base.Installer): """Additional tasks.""" install("spamassassin", self.config, self.upgrade, self.restore) install("clamav", self.config, self.upgrade, self.restore) + + def custom_backup(self, path): + """Backup custom configuration if any.""" + if package.backend.FORMAT == "deb": + amavis_custom = f"{self.config_dir}/conf.d/99-custom" + if os.path.isfile(amavis_custom): + utils.copy_file(amavis_custom, path) + utils.success("Amavis custom configuration saved!") + backup("spamassassin", path) + + def restore(self): + """Restore custom config files.""" + if package.backend.FORMAT != "deb": + return + amavis_custom_configuration = os.path.join( + self.restore, "custom/99-custom") + if os.path.isfile(amavis_custom_configuration): + utils.copy_file(amavis_custom_configuration, os.path.join( + self.config_dir, "conf.d")) + utils.success("Custom amavis configuration restored.") diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 3a1505b..2022ecd 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -54,6 +54,19 @@ class Installer(object): """Return a schema to install.""" return None + def get_sql_schema_from_backup(self): + """Retrieve a dump path from a previous backup.""" + utils.printcolor( + f"Trying to restore {self.appname} database from backup.", + utils.MAGENTA + ) + database_backup_path = os.path.join( + self.restore, f"databases/{self.appname}.sql") + if os.path.isfile(database_backup_path): + utils.success(f"SQL dump found in backup for {self.appname}!") + return database_backup_path + return None + def get_file_path(self, fname): """Return the absolute path of this file.""" return os.path.abspath( @@ -67,7 +80,11 @@ class Installer(object): return self.backend.create_user(self.dbuser, self.dbpasswd) self.backend.create_database(self.dbname, self.dbuser) - schema = self.get_sql_schema_path() + schema = None + if self.restore: + schema = self.get_sql_schema_from_backup() + if not schema: + schema = self.get_sql_schema_path() if schema: self.backend.load_sql_file( self.dbname, self.dbuser, self.dbpasswd, schema) @@ -138,6 +155,16 @@ class Installer(object): dst = os.path.join(self.config_dir, dst) utils.copy_from_template(src, dst, context) + def backup(self, path): + if self.with_db: + self._dump_database(path) + custom_backup_path = os.path.join(path, "custom") + self.custom_backup(custom_backup_path) + + def restore(self): + """Restore from a previous backup.""" + pass + def get_daemon_name(self): """Return daemon name if defined.""" return self.daemon_name if self.daemon_name else self.appname @@ -157,22 +184,17 @@ class Installer(object): if not self.upgrade: self.setup_database() self.install_config_files() + if self.restore: + self.restore() self.post_run() self.restart_daemon() - def _restore_database_dump(self, app_name): - """Restore database dump from a dump.""" - - utils.printcolor( - f"Trying to restore {app_name} database from backup.", utils.MAGENTA) - database_backup_path = os.path.join( - self.restore, f"databases/{app_name}.sql") - if os.path.isfile(database_backup_path): - utils.printcolor( - f"{app_name.capitalize()} database backup found ! Restoring...", utils.GREEN) - return database_backup_path - utils.printcolor( - f"{app_name.capitalize()} database backup not found, creating empty database.", utils.RED) + def _dump_database(self, backup_path: str): + """Create a new database dump for this app.""" + target_dir = os.path.join(backup_path, "databases") + target_file = os.path.join(target_dir, f"{self.appname}.sql") + self.backend.dump_database( + self.dbname, self.dbuser, self.dbpasswd, target_file) def pre_run(self): """Tasks to execute before the installer starts.""" diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 6cb80c6..4d1b3d9 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -96,28 +96,6 @@ class Dovecot(base.Installer): def post_run(self): """Additional tasks.""" - if self.restore is not None: - mail_dir = os.path.join(self.restore, "mails/") - if len(os.listdir(mail_dir)) > 0: - utils.printcolor( - "Copying mail backup over dovecot directory.", utils.GREEN) - - if os.path.exists(self.home_dir): - shutil.rmtree(self.home_dir) - - shutil.copytree(mail_dir, self.home_dir) - # Resetting permission for vmail - for dirpath, dirnames, filenames in os.walk(self.home_dir): - shutil.chown(dirpath, self.user, self.user) - for filename in filenames: - shutil.chown(os.path.join(dirpath, filename), - self.user, self.user) - else: - utils.printcolor( - "It seems that mails were not backed up, skipping mail restoration.", - utils.MAGENTA - ) - if self.dbengine == "postgres": dbname = self.config.get("modoboa", "dbname") dbuser = self.config.get("modoboa", "dbuser") @@ -156,3 +134,37 @@ class Dovecot(base.Installer): "service {} {} > /dev/null 2>&1".format(self.appname, action), capture_output=False) system.enable_service(self.get_daemon_name()) + + def backup(self, path): + """Backup emails.""" + utils.printcolor("Backing up mails", utils.MAGENTA) + if not os.path.exists(self.home_dir) or os.path.isfile(self.home_dir): + utils.printcolor("Error backing up emails, provided path " + f" ({self.home_dir}) seems not right...", utils.RED) + return + + dst = os.path.join(path, "mails/") + if os.path.exists(dst): + shutil.rmtree(dst) + shutil.copytree(self.home_dir, dst) + utils.printcolor("Mail backup complete!", utils.GREEN) + + def restore(self): + """Restore emails.""" + mail_dir = os.path.join(self.restore, "mails/") + if len(os.listdir(mail_dir)) > 0: + utils.success("Copying mail backup over dovecot directory.") + if os.path.exists(self.home_dir): + shutil.rmtree(self.home_dir) + shutil.copytree(mail_dir, self.home_dir) + # Resetting permission for vmail + for dirpath, dirnames, filenames in os.walk(self.home_dir): + shutil.chown(dirpath, self.user, self.user) + for filename in filenames: + shutil.chown(os.path.join(dirpath, filename), + self.user, self.user) + else: + utils.printcolor( + "It seems that emails were not backed up, skipping restoration.", + utils.MAGENTA + ) diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 0bc43b2..ea1320c 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -192,14 +192,6 @@ class Modoboa(base.Installer): self.backend.grant_access( self.config.get("amavis", "dbname"), self.dbuser) - def get_sql_schema_path(self): - if self.restore is not None: - db_dump_path = self._restore_database_dump("modoboa") - if db_dump_path is not None: - return db_dump_path - - return super(Modoboa, self).get_sql_schema_path() - def get_packages(self): """Include extra packages if needed.""" packages = super(Modoboa, self).get_packages() diff --git a/modoboa_installer/scripts/opendkim.py b/modoboa_installer/scripts/opendkim.py index 1dec655..9ad394b 100644 --- a/modoboa_installer/scripts/opendkim.py +++ b/modoboa_installer/scripts/opendkim.py @@ -2,6 +2,7 @@ import os import pwd +import shutil import stat from .. import database @@ -121,3 +122,12 @@ class Opendkim(base.Installer): "s/^After=(.*)$/After=$1 {}/".format(dbservice)) utils.exec_cmd( "perl -pi -e '{}' /lib/systemd/system/opendkim.service".format(pattern)) + + def custom_backup(self, path): + """Backup DKIM keys.""" + storage_dir = self.config.get( + "opendkim", "keys_storage_dir", fallback="/var/lib/dkim") + if os.path.isdir(storage_dir): + shutil.copytree(storage_dir, os.path.join(path, "dkim")) + utils.printcolor( + "DKIM keys saved!", utils.GREEN) diff --git a/modoboa_installer/scripts/postfix.py b/modoboa_installer/scripts/postfix.py index fc1a8f2..ce656b0 100644 --- a/modoboa_installer/scripts/postfix.py +++ b/modoboa_installer/scripts/postfix.py @@ -10,7 +10,7 @@ from .. import package from .. import utils from . import base -from . import install +from . import backup, install class Postfix(base.Installer): @@ -98,3 +98,7 @@ class Postfix(base.Installer): # Postwhite install("postwhite", self.config, self.upgrade, self.restore) + + def backup(self, path): + """Launch postwhite backup.""" + backup("postwhite", path) diff --git a/modoboa_installer/scripts/postwhite.py b/modoboa_installer/scripts/postwhite.py index ec54639..650708a 100644 --- a/modoboa_installer/scripts/postwhite.py +++ b/modoboa_installer/scripts/postwhite.py @@ -45,17 +45,26 @@ class Postwhite(base.Installer): """Additionnal tasks.""" install_dir = "/usr/local/bin" self.install_from_archive(SPF_TOOLS_REPOSITORY, install_dir) - postw_dir = self.install_from_archive( + self.postw_dir = self.install_from_archive( POSTWHITE_REPOSITORY, install_dir) - # Attempt to restore config file from backup - if self.restore is not None: - postwhite_backup_configuration = os.path.join( - self.restore, "custom/postwhite.conf") - if os.path.isfile(postwhite_backup_configuration): - utils.copy_file(postwhite_backup_configuration, self.config_dir) - utils.printcolor( - "postwhite.conf restored from backup", utils.GREEN) - else: - utils.copy_file(os.path.join(postw_dir, "postwhite.conf"), self.config_dir) - postw_bin = os.path.join(postw_dir, "postwhite") + postw_bin = os.path.join(self.postw_dir, "postwhite") utils.exec_cmd("{} /etc/postwhite.conf".format(postw_bin)) + + def custom_backup(self, path): + """Backup custom configuration if any.""" + postswhite_custom = "/etc/postwhite.conf" + if os.path.isfile(postswhite_custom): + utils.copy_file(postswhite_custom, path) + utils.printcolor( + "Postwhite configuration saved!", utils.GREEN) + + def restore(self): + """Restore config files.""" + postwhite_backup_configuration = os.path.join( + self.restore, "custom/postwhite.conf") + if os.path.isfile(postwhite_backup_configuration): + utils.copy_file(postwhite_backup_configuration, self.config_dir) + utils.success("postwhite.conf restored from backup") + else: + utils.copy_file( + os.path.join(self.postw_dir, "postwhite.conf"), self.config_dir) diff --git a/modoboa_installer/scripts/radicale.py b/modoboa_installer/scripts/radicale.py index df6e38b..fa20959 100644 --- a/modoboa_installer/scripts/radicale.py +++ b/modoboa_installer/scripts/radicale.py @@ -71,18 +71,18 @@ class Radicale(base.Installer): stat.S_IROTH | stat.S_IXOTH, 0, 0 ) - # Attempt to restore radicale collections from backup - if self.restore is not None: - radicale_backup = os.path.join( - self.restore, "custom/radicale") - if os.path.isdir(radicale_backup): - restore_target = os.path.join(self.home_dir, "collections") - if os.path.isdir(restore_target): - shutil.rmtree(restore_target) - shutil.copytree(radicale_backup, restore_target) - utils.printcolor( - "Radicale collections restored from backup", utils.GREEN) - super(Radicale, self).install_config_files() + super().install_config_files() + + def restore(self): + """Restore collections.""" + radicale_backup = os.path.join( + self.restore, "custom/radicale") + if os.path.isdir(radicale_backup): + restore_target = os.path.join(self.home_dir, "collections") + if os.path.isdir(restore_target): + shutil.rmtree(restore_target) + shutil.copytree(radicale_backup, restore_target) + utils.success("Radicale collections restored from backup") def post_run(self): """Additional tasks.""" @@ -93,3 +93,12 @@ class Radicale(base.Installer): system.enable_service(daemon_name) utils.exec_cmd("service {} stop".format(daemon_name)) utils.exec_cmd("service {} start".format(daemon_name)) + + def custom_backup(self, path): + """Backup collections.""" + radicale_backup = os.path.join(self.config.get( + "radicale", "home_dir", fallback="/srv/radicale"), "collections") + if os.path.isdir(radicale_backup): + shutil.copytree(radicale_backup, os.path.join( + path, "radicale")) + utils.printcolor("Radicale files saved", utils.GREEN) diff --git a/modoboa_installer/scripts/spamassassin.py b/modoboa_installer/scripts/spamassassin.py index f47ebd8..dede7e6 100644 --- a/modoboa_installer/scripts/spamassassin.py +++ b/modoboa_installer/scripts/spamassassin.py @@ -25,10 +25,6 @@ class Spamassassin(base.Installer): def get_sql_schema_path(self): """Return SQL schema.""" - if self.restore is not None: - db_dump_path = self._restore_database_dump("spamassassin") - if db_dump_path is not None: - return db_dump_path if self.dbengine == "postgres": fname = "bayes_pg.sql" else: diff --git a/modoboa_installer/utils.py b/modoboa_installer/utils.py index 827c650..5f97ccb 100644 --- a/modoboa_installer/utils.py +++ b/modoboa_installer/utils.py @@ -226,6 +226,16 @@ def printcolor(message, color): print(message) +def error(message): + """Print error message.""" + printcolor(message, RED) + + +def success(message): + """Print success message.""" + printcolor(message, GREEN) + + def convert_version_to_int(version): """Convert a version string to an integer.""" number_bits = (8, 8, 16) @@ -335,6 +345,57 @@ def gen_config(dest, interactive=False): current_username = getpass.getuser() current_user = pwd.getpwnam(current_username) os.chown(dest, current_user[2], current_user[3]) - os.chmod(dest, stat.S_IRUSR|stat.S_IWUSR) + os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) - \ No newline at end of file + +def validate_backup_path(path: str, silent_mode: bool): + """Check if provided backup path is valid or not.""" + path_exists = os.path.exists(path) + if path_exists and os.path.isfile(path): + printcolor( + "Error, you provided a file instead of a directory!", RED) + return None + + if not path_exists: + if not silent_mode: + create_dir = input( + f"\"{path}\" doesn't exist, would you like to create it? [Y/n]\n" + ).lower() + + if silent_mode or (not silent_mode and create_dir.startswith("y")): + pw = pwd.getpwnam("root") + mkdir_safe(path, stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3]) + else: + printcolor( + "Error, backup directory not present.", RED + ) + return None + + if len(os.listdir(path)) != 0: + if not silent_mode: + delete_dir = input( + "Warning: backup directory is not empty, it will be purged if you continue... [Y/n]\n").lower() + + if silent_mode or (not silent_mode and delete_dir.startswith("y")): + try: + os.remove(os.path.join(path, "installer.cfg")) + except FileNotFoundError: + pass + + shutil.rmtree(os.path.join(path, "custom"), + ignore_errors=False) + shutil.rmtree(os.path.join(path, "mails"), ignore_errors=False) + shutil.rmtree(os.path.join(path, "databases"), + ignore_errors=False) + else: + printcolor( + "Error: backup directory not clean.", RED + ) + return None + + backup_path = path + pw = pwd.getpwnam("root") + for dir in ["custom/", "databases/"]: + mkdir_safe(os.path.join(backup_path, dir), + stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3]) + return backup_path diff --git a/run.py b/run.py index c07040e..6a9a21a 100755 --- a/run.py +++ b/run.py @@ -3,6 +3,7 @@ """An installer for Modoboa.""" import argparse +import datetime import os try: import configparser @@ -11,6 +12,7 @@ except ImportError: import sys from modoboa_installer import compatibility_matrix +from modoboa_installer import constants from modoboa_installer import package from modoboa_installer import scripts from modoboa_installer import ssl @@ -18,6 +20,19 @@ from modoboa_installer import system from modoboa_installer import utils +PRIMARY_APPS = [ + "amavis", + "modoboa", + "automx", + "radicale", + "uwsgi", + "nginx", + "opendkim", + "postfix", + "dovecot" +] + + def installation_disclaimer(args, config): """Display installation disclaimer.""" hostname = config.get("general", "hostname") @@ -58,10 +73,46 @@ def restore_disclaimer(): """Display restore disclamer. """ utils.printcolor( "You are about to restore a previous installation of Modoboa." - "If a new version has been released in between, please update your database !", + "If a new version has been released in between, please update your database!", utils.BLUE) +def backup_system(config, args): + """Launch backup procedure.""" + backup_disclaimer() + backup_path = None + if args.silent_backup: + if not args.backup_path: + if config.has_option("backup", "default_path"): + path = config.get("backup", "default_path") + else: + path = constants.DEFAULT_BACKUP_DIRECTORY + date = datetime.datetime.now().strftime("%m_%d_%Y_%H_%M") + path = os.path.join(path, f"backup_{date}") + else: + path = args.backup_path + backup_path = utils.validate_backup_path(path) + if not backup_path: + utils.printcolor(f"Path provided: {path}", utils.BLUE) + return + else: + user_value = None + while not user_value or not backup_path: + utils.printcolor( + "Enter backup path (it must be an empty directory)", + utils.MAGENTA + ) + utils.printcolor("CTRL+C to cancel", utils.MAGENTA) + user_value = utils.user_input("-> ") + backup_path = utils.validate_backup_path(user_value) + + # Backup configuration file + utils.copy_file(args.configfile, backup_path) + # Backup applications + for app in PRIMARY_APPS: + scripts.backup(app, config, backup_path) + + def main(input_args): """Install process.""" parser = argparse.ArgumentParser() @@ -98,9 +149,6 @@ def main(input_args): "--silent-backup", action="store_true", default=False, help="For script usage, do not require user interaction " "backup will be saved at ./modoboa_backup/Backup_M_Y_d_H_M if --backup-path is not provided") - parser.add_argument( - "--no-mail-backup", action="store_true", default=False, - help="Disable mail backup (save space)") parser.add_argument( "--restore", type=str, metavar="path", help="Restore a previously backup up modoboa instance on a NEW machine. " @@ -117,17 +165,19 @@ def main(input_args): is_restoring = False if args.restore is not None: is_restoring = True - args.configfile = os.path.join(args.restore, "installer.cfg") + args.configfile = os.path.join(args.restore, args.configfile) if not os.path.exists(args.configfile): - utils.printcolor("installer.cfg from backup not found!", utils.RED) + utils.error( + "Installer configuration file not found in backup!" + ) sys.exit(1) - utils.printcolor("Welcome to Modoboa installer!\n", utils.GREEN) + utils.success("Welcome to Modoboa installer!\n") is_config_file_available = utils.check_config_file( args.configfile, args.interactive, args.upgrade, args.backup, is_restoring) if is_config_file_available and args.backup: - utils.printcolor("No config file found,", utils.RED) + utils.error("No config file found,") return if args.stop_after_configfile_check: @@ -142,14 +192,14 @@ def main(input_args): config.set("dovecot", "domain", args.domain) config.set("modoboa", "version", args.version) config.set("modoboa", "install_beta", str(args.beta)) + + if args.backup or args.silent_backup: + backup_system(config, args) + return + # Display disclaimer python 3 linux distribution if args.upgrade: upgrade_disclaimer(config) - elif args.backup or args.silent_backup: - backup_disclaimer() - scripts.backup(config, args.silent_backup, - args.backup_path, args.no_mail_backup) - return elif args.restore: restore_disclaimer() scripts.restore_prep(args.restore) @@ -175,33 +225,26 @@ def main(input_args): utils.printcolor( "The process can be long, feel free to take a coffee " "and come back later ;)", utils.BLUE) - utils.printcolor("Starting...", utils.GREEN) + utils.success("Starting...") package.backend.prepare_system() package.backend.install_many(["sudo", "wget"]) ssl_backend = ssl.get_backend(config) if ssl_backend and not args.upgrade: ssl_backend.generate_cert() - scripts.install("amavis", config, args.upgrade, args.restore) - scripts.install("modoboa", config, args.upgrade, args.restore) - scripts.install("automx", config, args.upgrade, args.restore) - scripts.install("radicale", config, args.upgrade, args.restore) - scripts.install("uwsgi", config, args.upgrade, args.restore) - scripts.install("nginx", config, args.upgrade, args.restore) - scripts.install("opendkim", config, args.upgrade, args.restore) - scripts.install("postfix", config, args.upgrade, args.restore) - scripts.install("dovecot", config, args.upgrade, args.restore) + for appname in PRIMARY_APPS: + scripts.install(appname, config, args.upgrade, args.restore) system.restart_service("cron") package.backend.restore_system() if not args.restore: - utils.printcolor( + utils.success( "Congratulations! You can enjoy Modoboa at https://{} (admin:password)" - .format(config.get("general", "hostname")), - utils.GREEN) + .format(config.get("general", "hostname")) + ) else: - utils.printcolor( + utils.success( "Restore complete! You can enjoy Modoboa at https://{} (same credentials as before)" - .format(config.get("general", "hostname")), - utils.GREEN) + .format(config.get("general", "hostname")) + ) if __name__ == "__main__": From 8b1d60ee596b57fc3dd0b3de21b7a6f82b9f4899 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Tue, 8 Nov 2022 17:19:23 +0100 Subject: [PATCH 2/4] Few fixes --- modoboa_installer/scripts/amavis.py | 2 +- modoboa_installer/scripts/base.py | 4 ++++ modoboa_installer/scripts/dovecot.py | 20 +++++++++++--------- modoboa_installer/scripts/postfix.py | 2 +- run.py | 20 ++++++++++++-------- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/modoboa_installer/scripts/amavis.py b/modoboa_installer/scripts/amavis.py index 4208bb0..662d453 100644 --- a/modoboa_installer/scripts/amavis.py +++ b/modoboa_installer/scripts/amavis.py @@ -102,7 +102,7 @@ class Amavis(base.Installer): if os.path.isfile(amavis_custom): utils.copy_file(amavis_custom, path) utils.success("Amavis custom configuration saved!") - backup("spamassassin", path) + backup("spamassassin", self.config, os.path.dirname(path)) def restore(self): """Restore custom config files.""" diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 2022ecd..00b68fe 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -161,6 +161,10 @@ class Installer(object): custom_backup_path = os.path.join(path, "custom") self.custom_backup(custom_backup_path) + def custom_backup(self, path): + """Override this method in subscripts to add custom backup content.""" + pass + def restore(self): """Restore from a previous backup.""" pass diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 4d1b3d9..628d0c6 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -137,28 +137,30 @@ class Dovecot(base.Installer): def backup(self, path): """Backup emails.""" + home_dir = self.config.get("dovecot", "home_dir") utils.printcolor("Backing up mails", utils.MAGENTA) - if not os.path.exists(self.home_dir) or os.path.isfile(self.home_dir): - utils.printcolor("Error backing up emails, provided path " - f" ({self.home_dir}) seems not right...", utils.RED) + if not os.path.exists(home_dir) or os.path.isfile(home_dir): + utils.error("Error backing up emails, provided path " + f" ({home_dir}) seems not right...") return dst = os.path.join(path, "mails/") if os.path.exists(dst): shutil.rmtree(dst) - shutil.copytree(self.home_dir, dst) - utils.printcolor("Mail backup complete!", utils.GREEN) + shutil.copytree(home_dir, dst) + utils.success("Mail backup complete!") def restore(self): """Restore emails.""" + home_dir = self.config.get("dovecot", "home_dir") mail_dir = os.path.join(self.restore, "mails/") if len(os.listdir(mail_dir)) > 0: utils.success("Copying mail backup over dovecot directory.") - if os.path.exists(self.home_dir): - shutil.rmtree(self.home_dir) - shutil.copytree(mail_dir, self.home_dir) + if os.path.exists(home_dir): + shutil.rmtree(home_dir) + shutil.copytree(mail_dir, home_dir) # Resetting permission for vmail - for dirpath, dirnames, filenames in os.walk(self.home_dir): + for dirpath, dirnames, filenames in os.walk(home_dir): shutil.chown(dirpath, self.user, self.user) for filename in filenames: shutil.chown(os.path.join(dirpath, filename), diff --git a/modoboa_installer/scripts/postfix.py b/modoboa_installer/scripts/postfix.py index ce656b0..52e9aa3 100644 --- a/modoboa_installer/scripts/postfix.py +++ b/modoboa_installer/scripts/postfix.py @@ -101,4 +101,4 @@ class Postfix(base.Installer): def backup(self, path): """Launch postwhite backup.""" - backup("postwhite", path) + backup("postwhite", self.config, path) diff --git a/run.py b/run.py index 6a9a21a..6420e88 100755 --- a/run.py +++ b/run.py @@ -64,9 +64,9 @@ def upgrade_disclaimer(config): def backup_disclaimer(): """Display backup disclamer. """ utils.printcolor( - "Your mail server will be backed up (messages and databases) locally." - " !! You should really transfer the backup somewhere else..." - " Custom configuration (like to postfix) won't be saved.", utils.BLUE) + "Your mail server will be backed up locally.\n" + " !! You should really transfer the backup somewhere else...\n" + " !! Custom configuration (like for postfix) won't be saved.", utils.BLUE) def restore_disclaimer(): @@ -91,7 +91,7 @@ def backup_system(config, args): path = os.path.join(path, f"backup_{date}") else: path = args.backup_path - backup_path = utils.validate_backup_path(path) + backup_path = utils.validate_backup_path(path, args.silent_backup) if not backup_path: utils.printcolor(f"Path provided: {path}", utils.BLUE) return @@ -104,7 +104,9 @@ def backup_system(config, args): ) utils.printcolor("CTRL+C to cancel", utils.MAGENTA) user_value = utils.user_input("-> ") - backup_path = utils.validate_backup_path(user_value) + if not user_value: + continue + backup_path = utils.validate_backup_path(user_value, args.silent_backup) # Backup configuration file utils.copy_file(args.configfile, backup_path) @@ -119,8 +121,6 @@ def main(input_args): versions = ( ["latest"] + list(compatibility_matrix.COMPATIBILITY_MATRIX.keys()) ) - parser.add_argument("--backup", action="store_true", default=False, - help="Backing up interactively previously installed instance") parser.add_argument("--debug", action="store_true", default=False, help="Enable debug output") parser.add_argument("--force", action="store_true", default=False, @@ -145,6 +145,10 @@ def main(input_args): parser.add_argument( "--backup-path", type=str, metavar="path", help="To use with --silent-backup, you must provide a valid path") + parser.add_argument( + "--backup", action="store_true", default=False, + help="Backing up interactively previously installed instance" + ) parser.add_argument( "--silent-backup", action="store_true", default=False, help="For script usage, do not require user interaction " @@ -176,7 +180,7 @@ def main(input_args): is_config_file_available = utils.check_config_file( args.configfile, args.interactive, args.upgrade, args.backup, is_restoring) - if is_config_file_available and args.backup: + if not is_config_file_available and (args.upgrade or args.backup): utils.error("No config file found,") return From d6f9a5b913aeca75bfc34e67c6787958334e08d2 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Tue, 8 Nov 2022 17:20:25 +0100 Subject: [PATCH 3/4] Few fixes. --- run.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run.py b/run.py index 6420e88..b4ac09d 100755 --- a/run.py +++ b/run.py @@ -180,8 +180,9 @@ def main(input_args): is_config_file_available = utils.check_config_file( args.configfile, args.interactive, args.upgrade, args.backup, is_restoring) - if not is_config_file_available and (args.upgrade or args.backup): - utils.error("No config file found,") + if not is_config_file_available and ( + args.upgrade or args.backup or args.silent_backup): + utils.error("No config file found.") return if args.stop_after_configfile_check: From 37633008cb05ebfe9c617c848e119d67fdd32e7e Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 9 Nov 2022 10:30:44 +0100 Subject: [PATCH 4/4] Fixed restore mode --- .github/workflows/installer.yml | 60 ++++++++++++++++++++++++++ .travis.yml | 15 ------- README.rst | 27 +++++------- modoboa_installer/scripts/__init__.py | 8 ++-- modoboa_installer/scripts/amavis.py | 6 +-- modoboa_installer/scripts/base.py | 12 +++--- modoboa_installer/scripts/dovecot.py | 2 +- modoboa_installer/scripts/opendkim.py | 26 +++++------ modoboa_installer/scripts/postfix.py | 2 +- modoboa_installer/scripts/postwhite.py | 2 +- modoboa_installer/scripts/radicale.py | 4 +- run.py | 2 +- 12 files changed, 103 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/installer.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml new file mode 100644 index 0000000..df9bfd6 --- /dev/null +++ b/.github/workflows/installer.yml @@ -0,0 +1,60 @@ +name: Modoboa installer + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + fail-fast: false + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install -r test-requirements.txt + - name: Run tests + if: ${{ matrix.python-version != '3.9' }} + run: | + python tests.py + - name: Run tests and coverage + if: ${{ matrix.python-version == '3.9' }} + run: | + coverage run tests.py + - name: Upload coverage result + if: ${{ matrix.python-version == '3.9' }} + uses: actions/upload-artifact@v2 + with: + name: coverage-results + path: .coverage + + coverage: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + - name: Install dependencies + run: | + pip install codecov + - name: Download coverage results + uses: actions/download-artifact@v2 + with: + name: coverage-results + - name: Report coverage + run: | + coverage report + codecov diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 641df12..0000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -sudo: false -language: python -cache: pip -python: - - "2.7" - - "3.4" - -before_install: - - pip install -r test-requirements.txt - -script: - - coverage run tests.py - -after_success: - - codecov diff --git a/README.rst b/README.rst index e2af8c7..5fa04ed 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ modoboa-installer ================= -|travis| |codecov| +|workflow| |codecov| An installer which deploy a complete mail server based on Modoboa. @@ -100,7 +100,7 @@ An experimental backup mode is available. .. warning:: You must keep the original configuration file, i.e. the one used for - the installation. Otherwise, you will need to recreate it manually with the right information ! + the installation. Otherwise, you will need to recreate it manually with the right information! You can start the process as follows:: @@ -108,7 +108,7 @@ You can start the process as follows:: Then follow the step on the console. -There are also a non-interactive mode: +There is also a non-interactive mode: 1. Silent mode @@ -116,21 +116,17 @@ Command:: $ sudo ./run.py --silent-backup -This mode is the silent batch mode, when executed, it will create /modoboa_backup/ and each time you execute it, it will create a new backup directory with current date and time. +This mode will run silently. When executed, it will create +/modoboa_backup/ and each time you execute it, it will create a new +backup directory with current date and time. -You can supply a custom path. - -Command:: +You can supply a custom path if needed:: $ sudo ./run.py --silent-backup --backup-path /path/of/backup/directory -This mode is the same as silent batch mode, but you provide the path to the backup directory you want. +If you want to disable emails backup, disable dovecot in the +configuration file (set enabled to False). - -If you want to disable mail backup:: - - $ sudo ./run.py {--backup|--silent-backup} --no-mail-backup - This can be useful for larger instance. Restore mode @@ -142,7 +138,7 @@ You can start the process as follows:: $ sudo ./run.py --restore /path/to/backup/directory/ -Then wait for the process to finish +Then wait for the process to finish. Change the generated hostname ----------------------------- @@ -187,7 +183,6 @@ modify the following settings:: Change the ``email`` setting to a valid value since it will be used for account recovery. -.. |travis| image:: https://travis-ci.org/modoboa/modoboa-installer.png?branch=master - :target: https://travis-ci.org/modoboa/modoboa-installer +.. |workflow| image:: https://github.com/modoboa/modoboa-installer/workflows/Modoboa%20installer/badge.svg .. |codecov| image:: http://codecov.io/github/modoboa/modoboa-installer/coverage.svg?branch=master :target: http://codecov.io/github/modoboa/modoboa-installer?branch=master diff --git a/modoboa_installer/scripts/__init__.py b/modoboa_installer/scripts/__init__.py index 8cf0047..d006f4c 100644 --- a/modoboa_installer/scripts/__init__.py +++ b/modoboa_installer/scripts/__init__.py @@ -17,7 +17,7 @@ def load_app_script(appname): return script -def install(appname, config, upgrade, restore): +def install(appname: str, config, upgrade: bool, archive_path: str): """Install an application.""" if (config.has_option(appname, "enabled") and not config.getboolean(appname, "enabled")): @@ -26,9 +26,9 @@ def install(appname, config, upgrade, restore): utils.printcolor("Installing {}".format(appname), utils.MAGENTA) script = load_app_script(appname) try: - getattr(script, appname.capitalize())(config, upgrade, restore).run() + getattr(script, appname.capitalize())(config, upgrade, archive_path).run() except utils.FatalError as inst: - utils.printcolor(u"{}".format(inst), utils.RED) + utils.error("{}".format(inst)) sys.exit(1) @@ -43,7 +43,7 @@ def backup(appname, config, path): try: getattr(script, appname.capitalize())(config, False, False).backup(path) except utils.FatalError as inst: - utils.printcolor(u"{}".format(inst), utils.RED) + utils.error("{}".format(inst)) sys.exit(1) diff --git a/modoboa_installer/scripts/amavis.py b/modoboa_installer/scripts/amavis.py index 662d453..897fbda 100644 --- a/modoboa_installer/scripts/amavis.py +++ b/modoboa_installer/scripts/amavis.py @@ -92,8 +92,8 @@ class Amavis(base.Installer): def post_run(self): """Additional tasks.""" - install("spamassassin", self.config, self.upgrade, self.restore) - install("clamav", self.config, self.upgrade, self.restore) + install("spamassassin", self.config, self.upgrade, self.archive_path) + install("clamav", self.config, self.upgrade, self.archive_path) def custom_backup(self, path): """Backup custom configuration if any.""" @@ -109,7 +109,7 @@ class Amavis(base.Installer): if package.backend.FORMAT != "deb": return amavis_custom_configuration = os.path.join( - self.restore, "custom/99-custom") + self.archive_path, "custom/99-custom") if os.path.isfile(amavis_custom_configuration): utils.copy_file(amavis_custom_configuration, os.path.join( self.config_dir, "conf.d")) diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 00b68fe..dea0897 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -20,11 +20,11 @@ class Installer(object): with_db = False config_files = [] - def __init__(self, config, upgrade, restore): + def __init__(self, config, upgrade: bool, archive_path: str): """Get configuration.""" self.config = config self.upgrade = upgrade - self.restore = restore + self.archive_path = archive_path if self.config.has_section(self.appname): self.app_config = dict(self.config.items(self.appname)) self.dbengine = self.config.get("database", "engine") @@ -61,7 +61,7 @@ class Installer(object): utils.MAGENTA ) database_backup_path = os.path.join( - self.restore, f"databases/{self.appname}.sql") + self.archive_path, f"databases/{self.appname}.sql") if os.path.isfile(database_backup_path): utils.success(f"SQL dump found in backup for {self.appname}!") return database_backup_path @@ -81,7 +81,7 @@ class Installer(object): self.backend.create_user(self.dbuser, self.dbpasswd) self.backend.create_database(self.dbname, self.dbuser) schema = None - if self.restore: + if self.archive_path: schema = self.get_sql_schema_from_backup() if not schema: schema = self.get_sql_schema_path() @@ -188,9 +188,9 @@ class Installer(object): if not self.upgrade: self.setup_database() self.install_config_files() - if self.restore: - self.restore() self.post_run() + if self.archive_path: + self.restore() self.restart_daemon() def _dump_database(self, backup_path: str): diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 628d0c6..a1f5988 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -153,7 +153,7 @@ class Dovecot(base.Installer): def restore(self): """Restore emails.""" home_dir = self.config.get("dovecot", "home_dir") - mail_dir = os.path.join(self.restore, "mails/") + mail_dir = os.path.join(self.archive_path, "mails/") if len(os.listdir(mail_dir)) > 0: utils.success("Copying mail backup over dovecot directory.") if os.path.exists(home_dir): diff --git a/modoboa_installer/scripts/opendkim.py b/modoboa_installer/scripts/opendkim.py index 9ad394b..0379221 100644 --- a/modoboa_installer/scripts/opendkim.py +++ b/modoboa_installer/scripts/opendkim.py @@ -47,19 +47,7 @@ class Opendkim(base.Installer): stat.S_IROTH | stat.S_IXOTH, target[1], target[2] ) - # Restore dkim keys from backup if restoring - if self.restore is not None: - dkim_keys_backup = os.path.join( - self.restore, "custom/dkim") - if os.path.isdir(dkim_keys_backup): - for file in os.listdir(dkim_keys_backup): - file_path = os.path.join(dkim_keys_backup, file) - if os.path.isfile(file_path): - utils.copy_file(file_path, self.config.get( - "opendkim", "keys_storage_dir", fallback="/var/lib/dkim")) - utils.printcolor( - "DKIM keys restored from backup", utils.GREEN) - super(Opendkim, self).install_config_files() + super().install_config_files() def get_template_context(self): """Additional variables.""" @@ -123,6 +111,18 @@ class Opendkim(base.Installer): utils.exec_cmd( "perl -pi -e '{}' /lib/systemd/system/opendkim.service".format(pattern)) + def restore(self): + """Restore keys.""" + dkim_keys_backup = os.path.join( + self.archive_path, "custom/dkim") + if os.path.isdir(dkim_keys_backup): + for file in os.listdir(dkim_keys_backup): + file_path = os.path.join(dkim_keys_backup, file) + if os.path.isfile(file_path): + utils.copy_file(file_path, self.config.get( + "opendkim", "keys_storage_dir", fallback="/var/lib/dkim")) + utils.success("DKIM keys restored from backup") + def custom_backup(self, path): """Backup DKIM keys.""" storage_dir = self.config.get( diff --git a/modoboa_installer/scripts/postfix.py b/modoboa_installer/scripts/postfix.py index 52e9aa3..a14e234 100644 --- a/modoboa_installer/scripts/postfix.py +++ b/modoboa_installer/scripts/postfix.py @@ -97,7 +97,7 @@ class Postfix(base.Installer): utils.exec_cmd("postalias {}".format(aliases_file)) # Postwhite - install("postwhite", self.config, self.upgrade, self.restore) + install("postwhite", self.config, self.upgrade, self.archive_path) def backup(self, path): """Launch postwhite backup.""" diff --git a/modoboa_installer/scripts/postwhite.py b/modoboa_installer/scripts/postwhite.py index 650708a..30bcb14 100644 --- a/modoboa_installer/scripts/postwhite.py +++ b/modoboa_installer/scripts/postwhite.py @@ -61,7 +61,7 @@ class Postwhite(base.Installer): def restore(self): """Restore config files.""" postwhite_backup_configuration = os.path.join( - self.restore, "custom/postwhite.conf") + self.archive_path, "custom/postwhite.conf") if os.path.isfile(postwhite_backup_configuration): utils.copy_file(postwhite_backup_configuration, self.config_dir) utils.success("postwhite.conf restored from backup") diff --git a/modoboa_installer/scripts/radicale.py b/modoboa_installer/scripts/radicale.py index fa20959..a0ca309 100644 --- a/modoboa_installer/scripts/radicale.py +++ b/modoboa_installer/scripts/radicale.py @@ -26,7 +26,7 @@ class Radicale(base.Installer): def __init__(self, *args, **kwargs): """Get configuration.""" - super(Radicale, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.venv_path = self.config.get("radicale", "venv_path") def _setup_venv(self): @@ -76,7 +76,7 @@ class Radicale(base.Installer): def restore(self): """Restore collections.""" radicale_backup = os.path.join( - self.restore, "custom/radicale") + self.archive_path, "custom/radicale") if os.path.isdir(radicale_backup): restore_target = os.path.join(self.home_dir, "collections") if os.path.isdir(restore_target): diff --git a/run.py b/run.py index b4ac09d..88c7ed6 100755 --- a/run.py +++ b/run.py @@ -72,7 +72,7 @@ def backup_disclaimer(): def restore_disclaimer(): """Display restore disclamer. """ utils.printcolor( - "You are about to restore a previous installation of Modoboa." + "You are about to restore a previous installation of Modoboa.\n" "If a new version has been released in between, please update your database!", utils.BLUE)