Merge pull request #424 from Spitfireap/restore

Backup & restore system

modoboa_installer/scripts/__init__.py

@@ -6,11 +6,12 @@ import sys
 from .. import utils
 
 
-def install(appname, config, upgrade):
+def install(appname, config, upgrade, restore):
     """Install an application."""
     if (config.has_option(appname, "enabled") and
             not config.getboolean(appname, "enabled")):
         return
+
     utils.printcolor("Installing {}".format(appname), utils.MAGENTA)
     try:
         script = importlib.import_module(
@@ -19,7 +20,26 @@ def install(appname, config, upgrade):
         print("Unknown application {}".format(appname))
         sys.exit(1)
     try:
-        getattr(script, appname.capitalize())(config, upgrade).run()
+        getattr(script, appname.capitalize())(config, upgrade, restore).run()
     except utils.FatalError as inst:
         utils.printcolor(u"{}".format(inst), utils.RED)
         sys.exit(1)
+
+
+def backup(config, silent_backup, backup_path, nomail):
+    """Backup instance"""
+    script = importlib.import_module(
+        "modoboa_installer.scripts.backup")
+    try:
+        getattr(script, "Backup")(
+            config, silent_backup, backup_path, nomail).run()
+    except utils.FatalError as inst:
+        utils.printcolor(u"{}".format(inst), utils.RED)
+        sys.exit(1)
+
+
+def restore_prep(restore):
+    """Restore instance"""
+    script = importlib.import_module(
+        "modoboa_installer.scripts.restore")
+    getattr(script, "Restore")(restore)

modoboa_installer/scripts/amavis.py

@@ -1,7 +1,6 @@
 """Amavis related functions."""
 
 import os
-import platform
 
 from .. import package
 from .. import utils
@@ -43,6 +42,14 @@ class Amavis(base.Installer):
     def get_config_files(self):
         """Return appropriate config files."""
         if package.backend.FORMAT == "deb":
+            if self.restore is not None:
+                amavis_custom_configuration = os.path.join(
+                    self.restore, "custom/99-custom")
+                if os.path.isfile(amavis_custom_configuration):
+                    utils.copy_file(amavis_custom_configuration, os.path.join(
+                        self.config_dir, "conf.d"))
+                    utils.printcolor(
+                        "Custom amavis configuration restored.", utils.GREEN)
             return [
                 "conf.d/05-node_id", "conf.d/15-content_filter_mode",
                 "conf.d/50-user"]
@@ -70,6 +77,11 @@ class Amavis(base.Installer):
 
     def get_sql_schema_path(self):
         """Return schema path."""
+        if self.restore:
+            db_dump_path = self._restore_database_dump("amavis")
+            if db_dump_path is not None:
+                return db_dump_path
+
         version = package.backend.get_installed_version("amavisd-new")
         if version is None:
             # Fallback to amavis...
@@ -83,7 +95,7 @@ class Amavis(base.Installer):
             path = self.get_file_path(
                 "amavis_{}_{}.sql".format(self.dbengine, version))
             if not os.path.exists(path):
-               raise utils.FatalError("Failed to find amavis database schema")
+                raise utils.FatalError("Failed to find amavis database schema")
         return path
 
     def pre_run(self):
@@ -93,5 +105,5 @@ class Amavis(base.Installer):
 
     def post_run(self):
         """Additional tasks."""
-        install("spamassassin", self.config, self.upgrade)
-        install("clamav", self.config, self.upgrade)
+        install("spamassassin", self.config, self.upgrade, self.restore)
+        install("clamav", self.config, self.upgrade, self.restore)

modoboa_installer/scripts/backup.py

@@ -0,0 +1,229 @@
+"""Backup script for pre-installed instance."""
+
+import os
+import pwd
+import shutil
+import stat
+import sys
+import datetime
+
+from .. import database
+from .. import utils
+from ..constants import DEFAULT_BACKUP_DIRECTORY
+
+
+class Backup:
+    """
+    Backup structure ( {optional} ):
+    {{backup_directory}}
+    ||
+    ||--> installer.cfg
+    ||--> custom
+        |--> { (copy of) /etc/amavis/conf.d/99-custom }
+        |--> { (copy of) /etc/postfix/custom_whitelist.cidr }
+        |--> { (copy of) dkim directory }
+            |--> {dkim.pem}...
+        |--> { (copy of) radicale home_dir }
+    ||--> databases
+        |--> modoboa.sql
+        |--> { amavis.sql }
+        |--> { spamassassin.sql }
+    ||--> mails
+        |--> vmails
+    """
+
+    def __init__(self, config, silent_backup, backup_path, nomail):
+        self.config = config
+        self.backup_path = backup_path
+        self.nomail = nomail
+        self.silent_backup = silent_backup
+
+    def validate_path(self, path):
+        """Check basic condition for backup directory."""
+
+        path_exists = os.path.exists(path)
+
+        if path_exists and os.path.isfile(path):
+            utils.printcolor(
+                "Error, you provided a file instead of a directory!", utils.RED)
+            return False
+
+        if not path_exists:
+            if not self.silent_backup:
+                create_dir = input(
+                    f"\"{path}\" doesn't exists, would you like to create it? [Y/n]\n").lower()
+
+            if self.silent_backup or (not self.silent_backup and create_dir.startswith("y")):
+                pw = pwd.getpwnam("root")
+                utils.mkdir_safe(path, stat.S_IRWXU |
+                                 stat.S_IRWXG, pw[2], pw[3])
+            else:
+                utils.printcolor(
+                    "Error, backup directory not present.", utils.RED
+                )
+                return False
+
+        if len(os.listdir(path)) != 0:
+            if not self.silent_backup:
+                delete_dir = input(
+                    "Warning : backup directory is not empty, it will be purged if you continue... [Y/n]\n").lower()
+
+            if self.silent_backup or (not self.silent_backup and (delete_dir == "y" or delete_dir == "yes")):
+                try:
+                    os.remove(os.path.join(path, "installer.cfg"))
+                except FileNotFoundError:
+                    pass
+
+                shutil.rmtree(os.path.join(path, "custom"),
+                              ignore_errors=False)
+                shutil.rmtree(os.path.join(path, "mails"), ignore_errors=False)
+                shutil.rmtree(os.path.join(path, "databases"),
+                              ignore_errors=False)
+            else:
+                utils.printcolor(
+                    "Error, backup dir not clean.", utils.RED
+                )
+                return False
+
+        self.backup_path = path
+
+        pw = pwd.getpwnam("root")
+        for dir in ["custom/", "databases/"]:
+            utils.mkdir_safe(os.path.join(self.backup_path, dir),
+                             stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3])
+        return True
+
+    def set_path(self):
+        """Setup backup directory."""
+        if self.silent_backup:
+            if self.backup_path is None:
+                if self.config.has_option("backup", "default_path"):
+                    path = self.config.get("backup", "default_path")
+                else:
+                    path = DEFAULT_BACKUP_DIRECTORY
+                date = datetime.datetime.now().strftime("%m_%d_%Y_%H_%M")
+                path = os.path.join(path, f"backup_{date}")
+                self.validate_path(path)
+            else:
+                if not self.validate_path(self.backup_path):
+                    utils.printcolor(
+                        f"Path provided : {self.backup_path}", utils.BLUE)
+                    sys.exit(1)
+        else:
+            user_value = None
+            while user_value == "" or user_value is None or not self.validate_path(user_value):
+                utils.printcolor(
+                    "Enter backup path (it must be an empty directory)", utils.MAGENTA)
+                utils.printcolor("CTRL+C to cancel", utils.MAGENTA)
+                user_value = utils.user_input("-> ")
+
+    def config_file_backup(self):
+        utils.copy_file("installer.cfg", self.backup_path)
+
+    def mail_backup(self):
+        if self.nomail:
+            utils.printcolor(
+                "Skipping mail backup, no-mail argument provided", utils.MAGENTA)
+            return
+
+        utils.printcolor("Backing up mails", utils.MAGENTA)
+
+        home_path = self.config.get("dovecot", "home_dir")
+
+        if not os.path.exists(home_path) or os.path.isfile(home_path):
+            utils.printcolor("Error backing up Email, provided path "
+                             f" ({home_path}) seems not right...", utils.RED)
+
+        else:
+            dst = os.path.join(self.backup_path, "mails/")
+
+            if os.path.exists(dst):
+                shutil.rmtree(dst)
+
+            shutil.copytree(home_path, dst)
+            utils.printcolor("Mail backup complete!", utils.GREEN)
+
+    def custom_config_backup(self):
+        """Custom config :
+        - DKIM keys: {{keys_storage_dir}}
+        - Radicale collection (calendat, contacts): {{home_dir}}
+        - Amavis : /etc/amavis/conf.d/99-custom
+        - Postwhite : /etc/postwhite.conf
+        Feel free to suggest to add others!"""
+        utils.printcolor(
+            "Backing up some custom configuration...", utils.MAGENTA)
+
+        custom_path = os.path.join(
+            self.backup_path, "custom")
+
+        # DKIM Key
+        if (self.config.has_option("opendkim", "enabled") and
+                self.config.getboolean("opendkim", "enabled")):
+            dkim_keys = self.config.get(
+                "opendkim", "keys_storage_dir", fallback="/var/lib/dkim")
+            if os.path.isdir(dkim_keys):
+                shutil.copytree(dkim_keys, os.path.join(custom_path, "dkim"))
+                utils.printcolor(
+                    "DKIM keys saved!", utils.GREEN)
+
+        # Radicale Collections
+        if (self.config.has_option("radicale", "enabled") and
+                self.config.getboolean("radicale", "enabled")):
+            radicale_backup = os.path.join(self.config.get(
+                "radicale", "home_dir", fallback="/srv/radicale"), "collections")
+            if os.path.isdir(radicale_backup):
+                shutil.copytree(radicale_backup, os.path.join(
+                    custom_path, "radicale"))
+                utils.printcolor("Radicale files saved", utils.GREEN)
+
+        # AMAVIS
+        if (self.config.has_option("amavis", "enabled") and
+                self.config.getboolean("amavis", "enabled")):
+            amavis_custom = "/etc/amavis/conf.d/99-custom"
+            if os.path.isfile(amavis_custom):
+                utils.copy_file(amavis_custom, custom_path)
+                utils.printcolor(
+                    "Amavis custom configuration saved!", utils.GREEN)
+
+        # POSTWHITE
+        if (self.config.has_option("postwhite", "enabled") and
+                self.config.getboolean("postwhite", "enabled")):
+            postswhite_custom = "/etc/postwhite.conf"
+            if os.path.isfile(postswhite_custom):
+                utils.copy_file(postswhite_custom, custom_path)
+                utils.printcolor(
+                    "Postwhite configuration saved!", utils.GREEN)
+
+    def database_backup(self):
+        """Backing up databases"""
+
+        utils.printcolor("Backing up databases...", utils.MAGENTA)
+
+        self.database_dump("modoboa")
+        self.database_dump("amavis")
+        self.database_dump("spamassassin")
+
+    def database_dump(self, app_name):
+
+        dump_path = os.path.join(self.backup_path, "databases")
+        backend = database.get_backend(self.config)
+
+        if app_name == "modoboa" or (self.config.has_option(app_name, "enabled") and
+                                     self.config.getboolean(app_name, "enabled")):
+            dbname = self.config.get(app_name, "dbname")
+            dbuser = self.config.get(app_name, "dbuser")
+            dbpasswd = self.config.get(app_name, "dbpassword")
+            backend.dump_database(dbname, dbuser, dbpasswd,
+                                  os.path.join(dump_path, f"{app_name}.sql"))
+
+    def backup_completed(self):
+        utils.printcolor("Backup process done, your backup is availible here:"
+                         f"--> {self.backup_path}", utils.GREEN)
+
+    def run(self):
+        self.set_path()
+        self.config_file_backup()
+        self.mail_backup()
+        self.custom_config_backup()
+        self.database_backup()
+        self.backup_completed()

modoboa_installer/scripts/base.py

@@ -20,10 +20,11 @@ class Installer(object):
     with_db = False
     config_files = []
 
-    def __init__(self, config, upgrade):
+    def __init__(self, config, upgrade, restore):
         """Get configuration."""
         self.config = config
         self.upgrade = upgrade
+        self.restore = restore
         if self.config.has_section(self.appname):
             self.app_config = dict(self.config.items(self.appname))
         self.dbengine = self.config.get("database", "engine")
@@ -159,6 +160,20 @@ class Installer(object):
         self.post_run()
         self.restart_daemon()
 
+    def _restore_database_dump(self, app_name):
+        """Restore database dump from a dump."""
+
+        utils.printcolor(
+                f"Trying to restore {app_name} database from backup.", utils.MAGENTA)
+        database_backup_path = os.path.join(
+            self.restore, f"databases/{app_name}.sql")
+        if os.path.isfile(database_backup_path):
+            utils.printcolor(
+                f"{app_name.capitalize()} database backup found ! Restoring...", utils.GREEN)
+            return database_backup_path
+        utils.printcolor(
+            f"{app_name.capitalize()} database backup not found, creating empty database.", utils.RED)
+
     def pre_run(self):
         """Tasks to execute before the installer starts."""
         pass

modoboa_installer/scripts/dovecot.py

@@ -3,6 +3,7 @@
 import glob
 import os
 import pwd
+import shutil
 
 from .. import database
 from .. import package
@@ -95,6 +96,25 @@ class Dovecot(base.Installer):
 
     def post_run(self):
         """Additional tasks."""
+        mail_dir = os.path.join(self.restore, "mails/")
+        if self.restore is not None and len(os.listdir(mail_dir)) > 0:
+            utils.printcolor(
+                "Copying mail backup over dovecot directory.", utils.GREEN)
+
+            if os.path.exists(self.home_dir):
+                shutil.rmtree(self.home_dir)
+
+            shutil.copytree(mail_dir, self.home_dir)
+            # Resetting permission for vmail
+            for dirpath, dirnames, filenames in os.walk(self.home_dir):
+                shutil.chown(dirpath, self.user, self.user)
+                for filename in filenames:
+                    shutil.chown(os.path.join(dirpath, filename),
+                                 self.user, self.user)
+        elif self.restore is not None:
+            utils.printcolor(
+                "It seems that mails were not backed up, skipping mail restoration.", utils.MAGENTA)
+
         if self.dbengine == "postgres":
             dbname = self.config.get("modoboa", "dbname")
             dbuser = self.config.get("modoboa", "dbuser")

modoboa_installer/scripts/modoboa.py

@@ -192,6 +192,14 @@ class Modoboa(base.Installer):
         self.backend.grant_access(
             self.config.get("amavis", "dbname"), self.dbuser)
 
+    def get_sql_schema_path(self):
+        if self.restore is not None:
+            db_dump_path = self._restore_database_dump("modoboa")
+            if db_dump_path is not None:
+                return db_dump_path
+
+        return super().get_sql_schema_path()()
+
     def get_packages(self):
         """Include extra packages if needed."""
         packages = super(Modoboa, self).get_packages()

modoboa_installer/scripts/nginx.py

@@ -26,7 +26,7 @@ class Nginx(base.Installer):
             "app_instance_path": (
                 self.config.get(app, "instance_path")),
             "uwsgi_socket_path": (
-                Uwsgi(self.config, self.upgrade).get_socket_path(app))
+                Uwsgi(self.config, self.upgrade, self.restore).get_socket_path(app))
         })
         return context
 

modoboa_installer/scripts/opendkim.py

@@ -46,6 +46,18 @@ class Opendkim(base.Installer):
                     stat.S_IROTH | stat.S_IXOTH,
                     target[1], target[2]
                 )
+        # Restore dkim keys from backup if restoring
+        if self.restore is not None:
+            dkim_keys_backup = os.path.join(
+                self.restore, "custom/dkim")
+            if os.path.isdir(dkim_keys_backup):
+                for file in os.listdir(dkim_keys_backup):
+                    file_path = os.path.join(dkim_keys_backup, file)
+                    if os.path.isfile(file_path):
+                        utils.copy_file(file_path, self.config.get(
+                            "opendkim", "keys_storage_dir", fallback="/var/lib/dkim"))
+                utils.printcolor(
+                    "DKIM keys restored from backup", utils.GREEN)
         super(Opendkim, self).install_config_files()
 
     def get_template_context(self):

modoboa_installer/scripts/postfix.py

@@ -97,4 +97,4 @@ class Postfix(base.Installer):
             utils.exec_cmd("postalias {}".format(aliases_file))
 
         # Postwhite
-        install("postwhite", self.config, self.upgrade)
+        install("postwhite", self.config, self.upgrade, self.restore)

modoboa_installer/scripts/postwhite.py

@@ -47,6 +47,15 @@ class Postwhite(base.Installer):
         self.install_from_archive(SPF_TOOLS_REPOSITORY, install_dir)
         postw_dir = self.install_from_archive(
             POSTWHITE_REPOSITORY, install_dir)
-        utils.copy_file(os.path.join(postw_dir, "postwhite.conf"), "/etc")
+        # Attempt to restore config file from backup
+        if self.restore is not None:
+            postwhite_backup_configuration = os.path.join(
+                self.restore, "custom/postwhite.conf")
+            if os.path.isfile(postwhite_backup_configuration):
+                utils.copy_file(postwhite_backup_configuration, self.config_dir)
+                utils.printcolor(
+                    "postwhite.conf restored from backup", utils.GREEN)
+            else:
+                utils.copy_file(os.path.join(postw_dir, "postwhite.conf"), self.config_dir)
         postw_bin = os.path.join(postw_dir, "postwhite")
         utils.exec_cmd("{} /etc/postwhite.conf".format(postw_bin))

modoboa_installer/scripts/radicale.py

@@ -1,6 +1,7 @@
 """Radicale related tasks."""
 
 import os
+import shutil
 import stat
 
 from .. import package
@@ -70,6 +71,17 @@ class Radicale(base.Installer):
                 stat.S_IROTH | stat.S_IXOTH,
                 0, 0
             )
+        # Attempt to restore radicale collections from backup
+        if self.restore is not None:
+            radicale_backup = os.path.join(
+                self.restore, "custom/radicale")
+            if os.path.isdir(radicale_backup):
+                restore_target = os.path.join(self.home_dir, "collections")
+                if os.path.isdir(restore_target):
+                    shutil.rmtree(restore_target)
+                shutil.copytree(radicale_backup, restore_target)
+                utils.printcolor(
+                    "Radicale collections restored from backup", utils.GREEN)
         super(Radicale, self).install_config_files()
 
     def post_run(self):

modoboa_installer/scripts/restore.py

@@ -0,0 +1,26 @@
+import os
+import sys
+from .. import utils
+
+
+class Restore:
+    def __init__(self, restore):
+        """
+        Restoring pre-check (backup integriety)
+        REQUIRED : modoboa.sql
+        OPTIONAL : mails/, custom/, amavis.sql, spamassassin.sql
+        Only checking required
+        """
+
+        if not os.path.isdir(restore):
+            utils.printcolor(
+                "Provided path is not a directory !", utils.RED)
+            sys.exit(1)
+
+        modoba_sql_file = os.path.join(restore, "databases/modoboa.sql")
+        if not os.path.isfile(modoba_sql_file):
+            utils.printcolor(
+                modoba_sql_file + " not found, please check your backup", utils.RED)
+            sys.exit(1)
+
+        # Everything seems allright here, proceding...

modoboa_installer/scripts/spamassassin.py

@@ -25,6 +25,10 @@ class Spamassassin(base.Installer):
 
     def get_sql_schema_path(self):
         """Return SQL schema."""
+        if self.restore is not None:
+            db_dump_path = self._restore_database_dump("spamassassin")
+            if db_dump_path is not None:
+                return db_dump_path
         if self.dbengine == "postgres":
             fname = "bayes_pg.sql"
         else:
@@ -63,7 +67,7 @@ class Spamassassin(base.Installer):
             "pyzor --homedir {} discover".format(pw[5]),
             sudo_user=amavis_user, login=False
         )
-        install("razor", self.config, self.upgrade)
+        install("razor", self.config, self.upgrade, self.restore)
         if utils.dist_name() in ["debian", "ubuntu"]:
             utils.exec_cmd(
                 "perl -pi -e 's/^CRON=0/CRON=1/' /etc/cron.daily/spamassassin")