Revert "Make pip quiet"

This reverts commit 7ccc871da7.

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [modoboa]

.github/workflows/installer.yml

@@ -0,0 +1,60 @@
+name: Modoboa installer
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install -r test-requirements.txt
+      - name: Run tests
+        if: ${{ matrix.python-version != '3.9' }}
+        run: |
+          python tests.py
+      - name: Run tests and coverage
+        if: ${{ matrix.python-version == '3.9' }}
+        run: |
+          coverage run tests.py
+      - name: Upload coverage result
+        if: ${{ matrix.python-version == '3.9' }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage-results
+          path: .coverage
+
+  coverage:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+      - name: Install dependencies
+        run: |
+          pip install codecov
+      - name: Download coverage results
+        uses: actions/download-artifact@v2
+        with:
+          name: coverage-results
+      - name: Report coverage
+        run: |
+          coverage report
+          codecov

.gitignore

@@ -55,3 +55,6 @@ docs/_build/
 
 # PyBuilder
 target/
+
+# PyCharm
+.idea/

.travis.yml

@@ -1,15 +0,0 @@
-sudo: false
-language: python
-cache: pip
-python:
-  - "2.7"
-  - "3.4"
-
-before_install:
-  - pip install -r test-requirements.txt
-
-script:
-  - coverage run tests.py
-
-after_success:
-  - codecov

README.rst

@@ -1,7 +1,7 @@
 modoboa-installer
 =================
 
-|travis| |codecov|
+|workflow| |codecov|
 
 An installer which deploy a complete mail server based on Modoboa.
 
@@ -92,6 +92,54 @@ You can activate it as follows::
 
 It will automatically install latest versions of modoboa and its plugins.
 
+Backup mode 
+------------
+
+An experimental backup mode is available.
+
+.. warning::
+
+   You must keep the original configuration file, i.e. the one used for
+   the installation. Otherwise, you will need to recreate it manually with the right information!
+
+You can start the process as follows::
+
+  $ sudo ./run.py --backup <your domain>
+
+Then follow the step on the console.
+
+There is also a non-interactive mode:
+
+1. Silent mode
+
+Command::
+
+  $ sudo ./run.py --silent-backup <your domain>
+
+This mode will run silently. When executed, it will create
+/modoboa_backup/ and each time you execute it, it will create a new
+backup directory with current date and time.
+
+You can supply a custom path if needed::
+
+  $ sudo ./run.py --silent-backup --backup-path /path/of/backup/directory <your domain>
+
+If you want to disable emails backup, disable dovecot in the
+configuration file (set enabled to False).
+
+This can be useful for larger instance.
+
+Restore mode
+------------
+
+An experimental restore mode is available.
+
+You can start the process as follows::
+
+  $ sudo ./run.py --restore /path/to/backup/directory/ <your domain>
+
+Then wait for the process to finish.
+
 Change the generated hostname
 -----------------------------
 
@@ -135,7 +183,6 @@ modify the following settings::
 Change the ``email`` setting to a valid value since it will be used
 for account recovery.
 
-.. |travis| image:: https://travis-ci.org/modoboa/modoboa-installer.png?branch=master
-   :target: https://travis-ci.org/modoboa/modoboa-installer
+.. |workflow| image:: https://github.com/modoboa/modoboa-installer/workflows/Modoboa%20installer/badge.svg
 .. |codecov| image:: http://codecov.io/github/modoboa/modoboa-installer/coverage.svg?branch=master
    :target: http://codecov.io/github/modoboa/modoboa-installer?branch=master

modoboa_installer/compatibility_matrix.py

@@ -20,7 +20,12 @@ COMPATIBILITY_MATRIX = {
         "modoboa-pdfcredentials": ">=1.1.1",
         "modoboa-sievefilters": ">=1.1.1",
         "modoboa-webmail": ">=1.2.0",
-    }
+    },
+    "2.1.0": {
+        "modoboa-pdfcredentials": None,
+        "modoboa-dmarc": None,
+        "modoboa-imap-migration": None,
+    },
 }
 
 EXTENSIONS_AVAILABILITY = {

modoboa_installer/config_dict_template.py

@@ -1,6 +1,8 @@
 import random
 import string
 
+from .constants import DEFAULT_BACKUP_DIRECTORY
+
 
 def make_password(length=16):
     """Create a random password."""
@@ -116,6 +118,31 @@ ConfigDictTemplate = [
             }
         ]
     },
+    {
+        "name": "fail2ban",
+        "values": [
+            {
+                "option": "enabled",
+                "default": "true",
+            },
+            {
+                "option": "config_dir",
+                "default": "/etc/fail2ban"
+            },
+            {
+                "option": "max_retry",
+                "default": "20"
+            },
+            {
+                "option": "ban_time",
+                "default": "3600"
+            },
+            {
+                "option": "find_time",
+                "default": "30"
+            },
+        ]
+    },
     {
         "name": "modoboa",
         "values": [
@@ -156,7 +183,7 @@ ConfigDictTemplate = [
             {
                 "option": "extensions",
                 "default": (
-                    "modoboa-amavis modoboa-pdfcredentials "
+                    "modoboa-amavis "
                     "modoboa-postfix-autoreply modoboa-sievefilters "
                     "modoboa-webmail modoboa-contacts "
                     "modoboa-radicale"
@@ -210,7 +237,7 @@ ConfigDictTemplate = [
             },
             {
                 "option": "max_servers",
-                "default": "1",
+                "default": "2",
             },
             {
                 "option": "dbname",
@@ -254,7 +281,7 @@ ConfigDictTemplate = [
             },
             {
                 "option": "user",
-                "default": "vmail",
+                "default": "dovecot",
             },
             {
                 "option": "home_dir",
@@ -319,6 +346,10 @@ ConfigDictTemplate = [
                 "option": "message_size_limit",
                 "default": "11534336",
             },
+            {
+                "option": "dhe_group",
+                "default": "4096"
+            }
         ]
     },
     {
@@ -439,4 +470,13 @@ ConfigDictTemplate = [
 
         ]
     },
+    {
+        "name": "backup",
+        "values": [
+            {
+                "option": "default_path",
+                "default": DEFAULT_BACKUP_DIRECTORY
+            }
+        ]
+    }
 ]

modoboa_installer/constants.py

@@ -0,0 +1 @@
+DEFAULT_BACKUP_DIRECTORY = "./modoboa_backup/"

modoboa_installer/database.py

@@ -146,6 +146,15 @@ class PostgreSQL(Database):
             self.dbhost, self.dbport, dbname, dbuser, path)
         utils.exec_cmd(cmd, sudo_user=self.dbuser)
 
+    def dump_database(self, dbname, dbuser, dbpassword, path):
+        """Dump DB to SQL file."""
+        # Reset pgpass since we backup multiple db (different secret set)
+        self._pgpass_done = False
+        self._setup_pgpass(dbname, dbuser, dbpassword)
+        cmd = "pg_dump -h {} -d {} -U {} -O  -w > {}".format(
+            self.dbhost, dbname, dbuser, path)
+        utils.exec_cmd(cmd, sudo_user=self.dbuser)
+
 
 class MySQL(Database):
 
@@ -169,12 +178,16 @@ class MySQL(Database):
         if name.startswith("debian"):
             if version.startswith("8"):
                 self.packages["deb"].append("libmysqlclient-dev")
-            elif version.startswith("11"):
+            elif version.startswith("11") or version.startswith("12"):
                 self.packages["deb"].append("libmariadb-dev")
             else:
                 self.packages["deb"].append("libmariadbclient-dev")
         elif name == "ubuntu":
-            self.packages["deb"].append("libmysqlclient-dev")
+            if version.startswith("2"):
+                # Works for Ubuntu 22 and 20
+                self.packages["deb"].append("libmariadb-dev")
+            else:
+                self.packages["deb"].append("libmysqlclient-dev")
         super(MySQL, self).install_package()
         queries = []
         if name.startswith("debian"):
@@ -186,12 +199,15 @@ class MySQL(Database):
                     "mariadb-server", "root_password_again", "password",
                     self.dbpassword)
                 return
-            if version.startswith("11"):
-                queries = [
-                    "SET PASSWORD FOR 'root'@'localhost' = PASSWORD('{}')"
-                    .format(self.dbpassword),
-                    "flush privileges"
-                ]
+        if (
+            (name.startswith("debian") and (version.startswith("11") or version.startswith("12"))) or
+            (name.startswith("ubuntu") and version.startswith("22"))
+        ):
+            queries = [
+                "SET PASSWORD FOR 'root'@'localhost' = PASSWORD('{}')"
+                .format(self.dbpassword),
+                "flush privileges"
+            ]
         if not queries:
             queries = [
                 "UPDATE user SET plugin='' WHERE user='root'",
@@ -258,6 +274,12 @@ class MySQL(Database):
                 self.dbhost, self.dbport, dbuser, dbpassword, dbname, path)
         )
 
+    def dump_database(self, dbname, dbuser, dbpassword, path):
+        """Dump DB to SQL file."""
+        cmd = "mysqldump -h {} -u {} -p{} {} > {}".format(
+            self.dbhost, dbuser, dbpassword, dbname, path)
+        utils.exec_cmd(cmd, sudo_user=self.dbuser)
+
 
 def get_backend(config):
     """Return appropriate backend."""

modoboa_installer/package.py

@@ -46,7 +46,7 @@ class DEBPackage(Package):
         """Update local cache."""
         if self.index_updated:
             return
-        utils.exec_cmd("apt-get update --quiet")
+        utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 update --quiet")
         self.index_updated = True
 
     def preconfigure(self, name, question, qtype, answer):
@@ -57,18 +57,18 @@ class DEBPackage(Package):
     def install(self, name):
         """Install a package."""
         self.update()
-        utils.exec_cmd("apt-get install --quiet --assume-yes {}".format(name))
+        utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 install --quiet --assume-yes {}".format(name))
 
     def install_many(self, names):
         """Install many packages."""
         self.update()
-        return utils.exec_cmd("apt-get install --quiet --assume-yes {}".format(
+        return utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 install --quiet --assume-yes {}".format(
             " ".join(names)))
 
     def get_installed_version(self, name):
         """Get installed package version."""
         code, output = utils.exec_cmd(
-            "dpkg -s {} | grep Version".format(name), capture_output=True)
+            "dpkg -s {} | grep Version".format(name))
         match = re.match(r"Version: (\d:)?(.+)-\d", output.decode())
         if match:
             return match.group(2)
@@ -97,7 +97,7 @@ class RPMPackage(Package):
     def get_installed_version(self, name):
         """Get installed package version."""
         code, output = utils.exec_cmd(
-            "rpm -qi {} | grep Version".format(name), capture_output=True)
+            "rpm -qi {} | grep Version".format(name))
         match = re.match(r"Version\s+: (.+)", output.decode())
         if match:
             return match.group(1)

modoboa_installer/python.py

@@ -1,6 +1,8 @@
 """Python related tools."""
 
+import json
 import os
+import sys
 
 from . import package
 from . import utils
@@ -45,6 +47,34 @@ def install_packages(names, venv=None, upgrade=False, **kwargs):
     utils.exec_cmd(cmd, **kwargs)
 
 
+def get_package_version(name, venv=None, **kwargs):
+    """Returns the version of an installed package."""
+    cmd = f"{get_pip_path(venv)} list --format json"
+    exit_code, output = utils.exec_cmd(cmd, **kwargs)
+    if exit_code != 0:
+        utils.error(f"Failed to get version of {name}. "
+                    f"Output is: {output}")
+        sys.exit(1)
+
+    print(f"name: {name}, venv: {venv}, cmd: {cmd}, exit_code: {exit_code}, output: {output.decode()}")
+    list_dict = json.loads(output.decode())
+    version_list = []
+    for element in list_dict:
+        if element["name"] == name:
+            version_list = element["version"].split(".")
+            break
+    version_list_clean = []
+    for element in version_list:
+        try:
+            version_list_clean.append(int(element))
+        except ValueError:
+            utils.printcolor(
+                f"Failed to decode some part of the version of {name}",
+                utils.YELLOW)
+            version_list_clean.append(element)
+    return version_list_clean
+
+
 def install_package_from_repository(name, url, vcs="git", venv=None, **kwargs):
     """Install a Python package from its repository."""
     if vcs == "git":
@@ -54,6 +84,16 @@ def install_package_from_repository(name, url, vcs="git", venv=None, **kwargs):
     utils.exec_cmd(cmd, **kwargs)
 
 
+def install_package_from_remote_requirements(url, venv=None, **kwargs):
+    """Install a Python package from a file."""
+    cmd = "{} install {} {}".format(
+        get_pip_path(venv),
+        "-r",
+        url
+    )
+    utils.exec_cmd(cmd, **kwargs)
+
+
 def setup_virtualenv(path, sudo_user=None, python_version=2):
     """Install a virtualenv if needed."""
     if os.path.exists(path):

modoboa_installer/scripts/__init__.py

@@ -6,20 +6,49 @@ import sys
 from .. import utils
 
 
-def install(appname, config, upgrade):
-    """Install an application."""
-    if (config.has_option(appname, "enabled") and
-            not config.getboolean(appname, "enabled")):
-        return
-    utils.printcolor("Installing {}".format(appname), utils.MAGENTA)
+def load_app_script(appname):
+    """Load module corresponding to the given appname."""
     try:
         script = importlib.import_module(
             "modoboa_installer.scripts.{}".format(appname))
     except ImportError:
         print("Unknown application {}".format(appname))
         sys.exit(1)
+    return script
+
+
+def install(appname: str, config, upgrade: bool, archive_path: str):
+    """Install an application."""
+    if (config.has_option(appname, "enabled") and
+            not config.getboolean(appname, "enabled")):
+        return
+
+    utils.printcolor("Installing {}".format(appname), utils.MAGENTA)
+    script = load_app_script(appname)
     try:
-        getattr(script, appname.capitalize())(config, upgrade).run()
+        getattr(script, appname.capitalize())(config, upgrade, archive_path).run()
     except utils.FatalError as inst:
-        utils.printcolor(u"{}".format(inst), utils.RED)
+        utils.error("{}".format(inst))
         sys.exit(1)
+
+
+def backup(appname, config, path):
+    """Backup an application."""
+    if (config.has_option(appname, "enabled") and
+            not config.getboolean(appname, "enabled")):
+        return
+
+    utils.printcolor("Backing up {}".format(appname), utils.MAGENTA)
+    script = load_app_script(appname)
+    try:
+        getattr(script, appname.capitalize())(config, False, False).backup(path)
+    except utils.FatalError as inst:
+        utils.error("{}".format(inst))
+        sys.exit(1)
+
+
+def restore_prep(restore):
+    """Restore instance"""
+    script = importlib.import_module(
+        "modoboa_installer.scripts.restore")
+    getattr(script, "Restore")(restore)

modoboa_installer/scripts/amavis.py

@@ -1,13 +1,12 @@
 """Amavis related functions."""
 
 import os
-import platform
 
 from .. import package
 from .. import utils
 
 from . import base
-from . import install
+from . import backup, install
 
 
 class Amavis(base.Installer):
@@ -83,7 +82,7 @@ class Amavis(base.Installer):
             path = self.get_file_path(
                 "amavis_{}_{}.sql".format(self.dbengine, version))
             if not os.path.exists(path):
-               raise utils.FatalError("Failed to find amavis database schema")
+                raise utils.FatalError("Failed to find amavis database schema")
         return path
 
     def pre_run(self):
@@ -93,5 +92,25 @@ class Amavis(base.Installer):
 
     def post_run(self):
         """Additional tasks."""
-        install("spamassassin", self.config, self.upgrade)
-        install("clamav", self.config, self.upgrade)
+        install("spamassassin", self.config, self.upgrade, self.archive_path)
+        install("clamav", self.config, self.upgrade, self.archive_path)
+
+    def custom_backup(self, path):
+        """Backup custom configuration if any."""
+        if package.backend.FORMAT == "deb":
+            amavis_custom = f"{self.config_dir}/conf.d/99-custom"
+            if os.path.isfile(amavis_custom):
+                utils.copy_file(amavis_custom, path)
+                utils.success("Amavis custom configuration saved!")
+        backup("spamassassin", self.config, os.path.dirname(path))
+
+    def restore(self):
+        """Restore custom config files."""
+        if package.backend.FORMAT != "deb":
+            return
+        amavis_custom_configuration = os.path.join(
+            self.archive_path, "custom/99-custom")
+        if os.path.isfile(amavis_custom_configuration):
+            utils.copy_file(amavis_custom_configuration, os.path.join(
+                self.config_dir, "conf.d"))
+            utils.success("Custom amavis configuration restored.")

modoboa_installer/scripts/automx.py

@@ -44,12 +44,12 @@ class Automx(base.Installer):
             sql_query = (
                 "SELECT first_name || ' ' || last_name AS display_name, email"
                 ", SPLIT_PART(email, '@', 2) AS domain "
-                "FROM core_user WHERE email='%s' AND is_active")
+                "FROM core_user WHERE email='%s' AND is_active;")
         else:
             sql_query = (
                 "SELECT concat(first_name, ' ', last_name) AS display_name, "
                 "email, SUBSTRING_INDEX(email, '@', -1) AS domain "
-                "FROM core_user WHERE email='%s' AND is_active=1"
+                "FROM core_user WHERE email='%s' AND is_active=1;"
             )
         context.update({"sql_dsn": sql_dsn, "sql_query": sql_query})
         return context
@@ -59,7 +59,7 @@ class Automx(base.Installer):
         python.setup_virtualenv(
             self.venv_path, sudo_user=self.user, python_version=3)
         packages = [
-            "future", "lxml", "ipaddress", "sqlalchemy", "python-memcached",
+            "future", "lxml", "ipaddress", "sqlalchemy < 2.0", "python-memcached",
             "python-dateutil", "configparser"
         ]
         if self.dbengine == "postgres":

modoboa_installer/scripts/backup.py

@@ -0,0 +1,226 @@
+"""Backup script for pre-installed instance."""
+
+import os
+import pwd
+import shutil
+import stat
+import sys
+import datetime
+
+from .. import database
+from .. import utils
+from ..constants import DEFAULT_BACKUP_DIRECTORY
+
+
+class Backup:
+    """
+    Backup structure ( {optional} ):
+    {{backup_directory}}
+    ||
+    ||--> installer.cfg
+    ||--> custom
+        |--> { (copy of) /etc/amavis/conf.d/99-custom }
+        |--> { (copy of) /etc/postfix/custom_whitelist.cidr }
+        |--> { (copy of) dkim directory }
+            |--> {dkim.pem}...
+        |--> { (copy of) radicale home_dir }
+    ||--> databases
+        |--> modoboa.sql
+        |--> { amavis.sql }
+        |--> { spamassassin.sql }
+    ||--> mails
+        |--> vmails
+    """
+
+    def __init__(self, config, silent_backup, backup_path, nomail):
+        self.config = config
+        self.backup_path = backup_path
+        self.nomail = nomail
+        self.silent_backup = silent_backup
+
+    def validate_path(self, path):
+        """Check basic condition for backup directory."""
+
+        path_exists = os.path.exists(path)
+
+        if path_exists and os.path.isfile(path):
+            utils.error("Error, you provided a file instead of a directory!")
+            return False
+
+        if not path_exists:
+            if not self.silent_backup:
+                create_dir = input(
+                    f"\"{path}\" doesn't exist, would you like to create it? [Y/n]\n").lower()
+
+            if self.silent_backup or (not self.silent_backup and create_dir.startswith("y")):
+                pw = pwd.getpwnam("root")
+                utils.mkdir_safe(path, stat.S_IRWXU |
+                                 stat.S_IRWXG, pw[2], pw[3])
+            else:
+                utils.error("Error, backup directory not present.")
+                return False
+
+        if len(os.listdir(path)) != 0:
+            if not self.silent_backup:
+                delete_dir = input(
+                    "Warning: backup directory is not empty, it will be purged if you continue... [Y/n]\n").lower()
+
+            if self.silent_backup or (not self.silent_backup and delete_dir.startswith("y")):
+                try:
+                    os.remove(os.path.join(path, "installer.cfg"))
+                except FileNotFoundError:
+                    pass
+
+                shutil.rmtree(os.path.join(path, "custom"),
+                              ignore_errors=False)
+                shutil.rmtree(os.path.join(path, "mails"), ignore_errors=False)
+                shutil.rmtree(os.path.join(path, "databases"),
+                              ignore_errors=False)
+            else:
+                utils.error("Error: backup directory not clean.")
+                return False
+
+        self.backup_path = path
+
+        pw = pwd.getpwnam("root")
+        for dir in ["custom/", "databases/"]:
+            utils.mkdir_safe(os.path.join(self.backup_path, dir),
+                             stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3])
+        return True
+
+    def set_path(self):
+        """Setup backup directory."""
+        if self.silent_backup:
+            if self.backup_path is None:
+                if self.config.has_option("backup", "default_path"):
+                    path = self.config.get("backup", "default_path")
+                else:
+                    path = DEFAULT_BACKUP_DIRECTORY
+                date = datetime.datetime.now().strftime("%m_%d_%Y_%H_%M")
+                path = os.path.join(path, f"backup_{date}")
+                self.validate_path(path)
+            else:
+                if not self.validate_path(self.backup_path):
+                    utils.printcolor(
+                        f"Path provided: {self.backup_path}", utils.BLUE)
+                    sys.exit(1)
+        else:
+            user_value = None
+            while user_value == "" or user_value is None or not self.validate_path(user_value):
+                utils.printcolor(
+                    "Enter backup path (it must be an empty directory)", utils.MAGENTA)
+                utils.printcolor("CTRL+C to cancel", utils.MAGENTA)
+                user_value = utils.user_input("-> ")
+
+    def config_file_backup(self):
+        utils.copy_file("installer.cfg", self.backup_path)
+
+    def mail_backup(self):
+        if self.nomail:
+            utils.printcolor(
+                "Skipping mail backup, no-mail argument provided", utils.MAGENTA)
+            return
+
+        utils.printcolor("Backing up mails", utils.MAGENTA)
+
+        home_path = self.config.get("dovecot", "home_dir")
+
+        if not os.path.exists(home_path) or os.path.isfile(home_path):
+            utils.error("Error backing up Email, provided path "
+                        f" ({home_path}) seems not right...")
+
+        else:
+            dst = os.path.join(self.backup_path, "mails/")
+
+            if os.path.exists(dst):
+                shutil.rmtree(dst)
+
+            shutil.copytree(home_path, dst)
+            utils.printcolor("Mail backup complete!", utils.GREEN)
+
+    def custom_config_backup(self):
+        """
+        Custom config :
+        - DKIM keys: {{keys_storage_dir}}
+        - Radicale collection (calendat, contacts): {{home_dir}}
+        - Amavis : /etc/amavis/conf.d/99-custom
+        - Postwhite : /etc/postwhite.conf
+        Feel free to suggest to add others!
+        """
+        utils.printcolor(
+            "Backing up some custom configuration...", utils.MAGENTA)
+
+        custom_path = os.path.join(
+            self.backup_path, "custom")
+
+        # DKIM Key
+        if (self.config.has_option("opendkim", "enabled") and
+                self.config.getboolean("opendkim", "enabled")):
+            dkim_keys = self.config.get(
+                "opendkim", "keys_storage_dir", fallback="/var/lib/dkim")
+            if os.path.isdir(dkim_keys):
+                shutil.copytree(dkim_keys, os.path.join(custom_path, "dkim"))
+                utils.printcolor(
+                    "DKIM keys saved!", utils.GREEN)
+
+        # Radicale Collections
+        if (self.config.has_option("radicale", "enabled") and
+                self.config.getboolean("radicale", "enabled")):
+            radicale_backup = os.path.join(self.config.get(
+                "radicale", "home_dir", fallback="/srv/radicale"), "collections")
+            if os.path.isdir(radicale_backup):
+                shutil.copytree(radicale_backup, os.path.join(
+                    custom_path, "radicale"))
+                utils.printcolor("Radicale files saved", utils.GREEN)
+
+        # AMAVIS
+        if (self.config.has_option("amavis", "enabled") and
+                self.config.getboolean("amavis", "enabled")):
+            amavis_custom = "/etc/amavis/conf.d/99-custom"
+            if os.path.isfile(amavis_custom):
+                utils.copy_file(amavis_custom, custom_path)
+                utils.printcolor(
+                    "Amavis custom configuration saved!", utils.GREEN)
+
+        # POSTWHITE
+        if (self.config.has_option("postwhite", "enabled") and
+                self.config.getboolean("postwhite", "enabled")):
+            postswhite_custom = "/etc/postwhite.conf"
+            if os.path.isfile(postswhite_custom):
+                utils.copy_file(postswhite_custom, custom_path)
+                utils.printcolor(
+                    "Postwhite configuration saved!", utils.GREEN)
+
+    def database_backup(self):
+        """Backing up databases"""
+
+        utils.printcolor("Backing up databases...", utils.MAGENTA)
+
+        self.database_dump("modoboa")
+        self.database_dump("amavis")
+        self.database_dump("spamassassin")
+
+    def database_dump(self, app_name):
+
+        dump_path = os.path.join(self.backup_path, "databases")
+        backend = database.get_backend(self.config)
+
+        if app_name == "modoboa" or (self.config.has_option(app_name, "enabled") and
+                                     self.config.getboolean(app_name, "enabled")):
+            dbname = self.config.get(app_name, "dbname")
+            dbuser = self.config.get(app_name, "dbuser")
+            dbpasswd = self.config.get(app_name, "dbpassword")
+            backend.dump_database(dbname, dbuser, dbpasswd,
+                                  os.path.join(dump_path, f"{app_name}.sql"))
+
+    def backup_completed(self):
+        utils.printcolor("Backup process done, your backup is available here:"
+                         f"--> {self.backup_path}", utils.GREEN)
+
+    def run(self):
+        self.set_path()
+        self.config_file_backup()
+        self.mail_backup()
+        self.custom_config_backup()
+        self.database_backup()
+        self.backup_completed()

modoboa_installer/scripts/base.py

@@ -5,11 +5,12 @@ import sys
 
 from .. import database
 from .. import package
+from .. import python
 from .. import system
 from .. import utils
 
 
-class Installer(object):
+class Installer:
     """Simple installer for one application."""
 
     appname = None
@@ -20,10 +21,11 @@ class Installer(object):
     with_db = False
     config_files = []
 
-    def __init__(self, config, upgrade):
+    def __init__(self, config, upgrade: bool, archive_path: str):
         """Get configuration."""
         self.config = config
         self.upgrade = upgrade
+        self.archive_path = archive_path
         if self.config.has_section(self.appname):
             self.app_config = dict(self.config.items(self.appname))
         self.dbengine = self.config.get("database", "engine")
@@ -41,6 +43,20 @@ class Installer(object):
         self.dbuser = self.config.get(self.appname, "dbuser")
         self.dbpasswd = self.config.get(self.appname, "dbpassword")
 
+    @property
+    def modoboa_2_2_or_greater(self):
+        # Check if modoboa version > 2.2
+        modoboa_version = python.get_package_version(
+            "modoboa",
+            self.config.get("modoboa", "venv_path"),
+            sudo_user=self.config.get("modoboa", "user")
+            )
+        condition = (
+            (int(modoboa_version[0]) == 2 and int(modoboa_version[1]) >= 2) or
+            int(modoboa_version[0]) > 2
+            )
+        return condition
+
     @property
     def config_dir(self):
         """Return main configuration directory."""
@@ -53,6 +69,19 @@ class Installer(object):
         """Return a schema to install."""
         return None
 
+    def get_sql_schema_from_backup(self):
+        """Retrieve a dump path from a previous backup."""
+        utils.printcolor(
+            f"Trying to restore {self.appname} database from backup.",
+            utils.MAGENTA
+        )
+        database_backup_path = os.path.join(
+            self.archive_path, f"databases/{self.appname}.sql")
+        if os.path.isfile(database_backup_path):
+            utils.success(f"SQL dump found in backup for {self.appname}!")
+            return database_backup_path
+        return None
+
     def get_file_path(self, fname):
         """Return the absolute path of this file."""
         return os.path.abspath(
@@ -66,7 +95,11 @@ class Installer(object):
             return
         self.backend.create_user(self.dbuser, self.dbpasswd)
         self.backend.create_database(self.dbname, self.dbuser)
-        schema = self.get_sql_schema_path()
+        schema = None
+        if self.archive_path:
+            schema = self.get_sql_schema_from_backup()
+        if not schema:
+            schema = self.get_sql_schema_path()
         if schema:
             self.backend.load_sql_file(
                 self.dbname, self.dbuser, self.dbpasswd, schema)
@@ -113,7 +146,7 @@ class Installer(object):
             return
         exitcode, output = package.backend.install_many(packages)
         if exitcode:
-            utils.printcolor("Failed to install dependencies", utils.RED)
+            utils.error("Failed to install dependencies")
             sys.exit(1)
 
     def get_config_files(self):
@@ -137,6 +170,20 @@ class Installer(object):
                 dst = os.path.join(self.config_dir, dst)
             utils.copy_from_template(src, dst, context)
 
+    def backup(self, path):
+        if self.with_db:
+            self._dump_database(path)
+        custom_backup_path = os.path.join(path, "custom")
+        self.custom_backup(custom_backup_path)
+
+    def custom_backup(self, path):
+        """Override this method in subscripts to add custom backup content."""
+        pass
+
+    def restore(self):
+        """Restore from a previous backup."""
+        pass
+
     def get_daemon_name(self):
         """Return daemon name if defined."""
         return self.daemon_name if self.daemon_name else self.appname
@@ -157,8 +204,17 @@ class Installer(object):
             self.setup_database()
         self.install_config_files()
         self.post_run()
+        if self.archive_path:
+            self.restore()
         self.restart_daemon()
 
+    def _dump_database(self, backup_path: str):
+        """Create a new database dump for this app."""
+        target_dir = os.path.join(backup_path, "databases")
+        target_file = os.path.join(target_dir, f"{self.appname}.sql")
+        self.backend.dump_database(
+            self.dbname, self.dbuser, self.dbpasswd, target_file)
+
     def pre_run(self):
         """Tasks to execute before the installer starts."""
         pass

modoboa_installer/scripts/clamav.py

@@ -57,7 +57,7 @@ class Clamav(base.Installer):
             # Check if not present before
             path = "/usr/lib/systemd/system/clamd@.service"
             code, output = utils.exec_cmd(
-                "grep 'WantedBy=multi-user.target' {}".format(path))
+                r"grep 'WantedBy\s*=\s*multi-user.target' {}".format(path))
             if code:
                 utils.exec_cmd(
                     """cat <<EOM >> {}

modoboa_installer/scripts/dovecot.py

@@ -3,6 +3,7 @@
 import glob
 import os
 import pwd
+import shutil
 
 from .. import database
 from .. import package
@@ -26,9 +27,15 @@ class Dovecot(base.Installer):
     }
     config_files = [
         "dovecot.conf", "dovecot-dict-sql.conf.ext", "conf.d/10-ssl.conf",
-        "conf.d/10-master.conf", "conf.d/20-lmtp.conf"]
+        "conf.d/10-master.conf", "conf.d/20-lmtp.conf", "conf.d/10-ssl-keys.try"]
     with_user = True
 
+    def setup_user(self):
+        """Setup mailbox user."""
+        super().setup_user()
+        self.mailboxes_owner = self.app_config["mailboxes_owner"]
+        system.create_user(self.mailboxes_owner, self.home_dir)
+
     def get_config_files(self):
         """Additional config files."""
         return self.config_files + [
@@ -57,10 +64,17 @@ class Dovecot(base.Installer):
     def get_template_context(self):
         """Additional variables."""
         context = super(Dovecot, self).get_template_context()
-        pw = pwd.getpwnam(self.user)
+        pw_mailbox = pwd.getpwnam(self.mailboxes_owner)
+        dovecot_package = {"deb": "dovecot-core", "rpm": "dovecot"}
+        ssl_protocol_parameter = "ssl_protocols"
+        if package.backend.get_installed_version(dovecot_package[package.backend.FORMAT]) > "2.3":
+            ssl_protocol_parameter = "ssl_min_protocol"
         ssl_protocols = "!SSLv2 !SSLv3"
-        if package.backend.get_installed_version("openssl").startswith("1.1"):
+        if package.backend.get_installed_version("openssl").startswith("1.1") \
+                or package.backend.get_installed_version("openssl").startswith("3"):
             ssl_protocols = "!SSLv3"
+        if ssl_protocol_parameter == "ssl_min_protocol":
+            ssl_protocols = "TLSv1"
         if "centos" in utils.dist_name():
             protocols = "protocols = imap lmtp sieve"
             extra_protocols = self.config.get("dovecot", "extra_protocols")
@@ -69,19 +83,24 @@ class Dovecot(base.Installer):
         else:
             # Protocols are automatically guessed on debian/ubuntu
             protocols = ""
+
         context.update({
             "db_driver": self.db_driver,
-            "mailboxes_owner_uid": pw[2],
-            "mailboxes_owner_gid": pw[3],
+            "mailboxes_owner_uid": pw_mailbox[2],
+            "mailboxes_owner_gid": pw_mailbox[3],
+            "mailbox_owner": self.mailboxes_owner,
             "modoboa_user": self.config.get("modoboa", "user"),
             "modoboa_dbname": self.config.get("modoboa", "dbname"),
             "modoboa_dbuser": self.config.get("modoboa", "dbuser"),
             "modoboa_dbpassword": self.config.get("modoboa", "dbpassword"),
             "protocols": protocols,
             "ssl_protocols": ssl_protocols,
+            "ssl_protocol_parameter": ssl_protocol_parameter,
             "radicale_user": self.config.get("radicale", "user"),
             "radicale_auth_socket_path": os.path.basename(
-                self.config.get("dovecot", "radicale_auth_socket_path"))
+                self.config.get("dovecot", "radicale_auth_socket_path")),
+            "modoboa_2_2_or_greater": "" if self.modoboa_2_2_or_greater else "#",
+            "not_modoboa_2_2_or_greater": "" if not self.modoboa_2_2_or_greater else "#"
         })
         return context
 
@@ -104,12 +123,12 @@ class Dovecot(base.Installer):
             utils.copy_file(f, "{}/conf.d".format(self.config_dir))
         # Make postlogin script executable
         utils.exec_cmd("chmod +x /usr/local/bin/postlogin.sh")
+        # Only root should have read access to the 10-ssl-keys.try
+        # See https://github.com/modoboa/modoboa/issues/2570
+        utils.exec_cmd("chmod 600 /etc/dovecot/conf.d/10-ssl-keys.try")
         # Add mailboxes user to dovecot group for modoboa mailbox commands.
         # See https://github.com/modoboa/modoboa/issues/2157.
-        system.add_user_to_group(
-            self.config.get("dovecot", "mailboxes_owner"),
-            'dovecot'
-        )
+        system.add_user_to_group(self.mailboxes_owner, 'dovecot')
 
     def restart_daemon(self):
         """Restart daemon process.
@@ -125,3 +144,39 @@ class Dovecot(base.Installer):
             "service {} {} > /dev/null 2>&1".format(self.appname, action),
             capture_output=False)
         system.enable_service(self.get_daemon_name())
+
+    def backup(self, path):
+        """Backup emails."""
+        home_dir = self.config.get("dovecot", "home_dir")
+        utils.printcolor("Backing up mails", utils.MAGENTA)
+        if not os.path.exists(home_dir) or os.path.isfile(home_dir):
+            utils.error("Error backing up emails, provided path "
+                        f" ({home_dir}) seems not right...")
+            return
+
+        dst = os.path.join(path, "mails/")
+        if os.path.exists(dst):
+            shutil.rmtree(dst)
+        shutil.copytree(home_dir, dst)
+        utils.success("Mail backup complete!")
+
+    def restore(self):
+        """Restore emails."""
+        home_dir = self.config.get("dovecot", "home_dir")
+        mail_dir = os.path.join(self.archive_path, "mails/")
+        if len(os.listdir(mail_dir)) > 0:
+            utils.success("Copying mail backup over dovecot directory.")
+            if os.path.exists(home_dir):
+                shutil.rmtree(home_dir)
+            shutil.copytree(mail_dir, home_dir)
+            # Resetting permission for vmail
+            for dirpath, dirnames, filenames in os.walk(home_dir):
+                shutil.chown(dirpath, self.mailboxes_owner, self.mailboxes_owner)
+                for filename in filenames:
+                    shutil.chown(os.path.join(dirpath, filename),
+                                 self.mailboxes_owner, self.mailboxes_owner)
+        else:
+            utils.printcolor(
+                "It seems that emails were not backed up, skipping restoration.",
+                utils.MAGENTA
+            )

modoboa_installer/scripts/fail2ban.py

@@ -0,0 +1,17 @@
+"""fail2ban related functions."""
+
+from . import base
+
+
+class Fail2ban(base.Installer):
+    """Fail2ban installer."""
+
+    appname = "fail2ban"
+    packages = {
+        "deb": ["fail2ban"],
+        "rpm": ["fail2ban"]
+    }
+    config_files = [
+        "jail.d/modoboa.conf",
+        "filter.d/modoboa-auth.conf",
+    ]

modoboa_installer/scripts/files/amavis/amavis_mysql_2.13.X.sql

@@ -0,0 +1,213 @@
+-- Amavis 2.11.0 MySQL schema
+-- Provided by Modoboa
+-- Warning: foreign key creations are enabled
+
+-- local users
+CREATE TABLE users (
+  id         int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY,  -- unique id
+  priority   integer      NOT NULL DEFAULT '7',  -- sort field, 0 is low prior.
+  policy_id  integer unsigned NOT NULL DEFAULT '1',  -- JOINs with policy.id
+  email      varbinary(255) NOT NULL UNIQUE,
+  fullname   varchar(255) DEFAULT NULL    -- not used by amavisd-new
+  -- local   char(1)      -- Y/N  (optional field, see note further down)
+);
+
+-- any e-mail address (non- rfc2822-quoted), external or local,
+-- used as senders in wblist
+CREATE TABLE mailaddr (
+  id         int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  priority   integer      NOT NULL DEFAULT '7',  -- 0 is low priority
+  email      varbinary(255) NOT NULL UNIQUE
+);
+
+-- per-recipient whitelist and/or blacklist,
+-- puts sender and recipient in relation wb  (white or blacklisted sender)
+CREATE TABLE wblist (
+  rid        integer unsigned NOT NULL,  -- recipient: users.id
+  sid        integer unsigned NOT NULL,  -- sender: mailaddr.id
+  wb         varchar(10)  NOT NULL,  -- W or Y / B or N / space=neutral / score
+  PRIMARY KEY (rid,sid)
+);
+
+CREATE TABLE policy (
+  id  int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY,
+                                    -- 'id' this is the _only_ required field
+  policy_name      varchar(32),     -- not used by amavisd-new, a comment
+
+  virus_lover           char(1) default NULL,     -- Y/N
+  spam_lover            char(1) default NULL,     -- Y/N
+  unchecked_lover       char(1) default NULL,     -- Y/N
+  banned_files_lover    char(1) default NULL,     -- Y/N
+  bad_header_lover      char(1) default NULL,     -- Y/N
+
+  bypass_virus_checks   char(1) default NULL,     -- Y/N
+  bypass_spam_checks    char(1) default NULL,     -- Y/N
+  bypass_banned_checks  char(1) default NULL,     -- Y/N
+  bypass_header_checks  char(1) default NULL,     -- Y/N
+
+  virus_quarantine_to      varchar(64) default NULL,
+  spam_quarantine_to       varchar(64) default NULL,
+  banned_quarantine_to     varchar(64) default NULL,
+  unchecked_quarantine_to  varchar(64) default NULL,
+  bad_header_quarantine_to varchar(64) default NULL,
+  clean_quarantine_to      varchar(64) default NULL,
+  archive_quarantine_to    varchar(64) default NULL,
+
+  spam_tag_level  float default NULL, -- higher score inserts spam info headers
+  spam_tag2_level float default NULL, -- inserts 'declared spam' header fields
+  spam_tag3_level float default NULL, -- inserts 'blatant spam' header fields
+  spam_kill_level float default NULL, -- higher score triggers evasive actions
+                                      -- e.g. reject/drop, quarantine, ...
+                                     -- (subject to final_spam_destiny setting)
+  spam_dsn_cutoff_level        float default NULL,
+  spam_quarantine_cutoff_level float default NULL,
+
+  addr_extension_virus      varchar(64) default NULL,
+  addr_extension_spam       varchar(64) default NULL,
+  addr_extension_banned     varchar(64) default NULL,
+  addr_extension_bad_header varchar(64) default NULL,
+
+  warnvirusrecip      char(1)     default NULL, -- Y/N
+  warnbannedrecip     char(1)     default NULL, -- Y/N
+  warnbadhrecip       char(1)     default NULL, -- Y/N
+  newvirus_admin      varchar(64) default NULL,
+  virus_admin         varchar(64) default NULL,
+  banned_admin        varchar(64) default NULL,
+  bad_header_admin    varchar(64) default NULL,
+  spam_admin          varchar(64) default NULL,
+  spam_subject_tag    varchar(64) default NULL,
+  spam_subject_tag2   varchar(64) default NULL,
+  spam_subject_tag3   varchar(64) default NULL,
+  message_size_limit  integer     default NULL, -- max size in bytes, 0 disable
+  banned_rulenames    varchar(64) default NULL, -- comma-separated list of ...
+        -- names mapped through %banned_rules to actual banned_filename tables
+  disclaimer_options  varchar(64) default NULL,
+  forward_method      varchar(64) default NULL,
+  sa_userconf         varchar(64) default NULL,
+  sa_username         varchar(64) default NULL
+);
+
+
+-- R/W part of the dataset (optional)
+--   May reside in the same or in a separate database as lookups database;
+--   REQUIRES SUPPORT FOR TRANSACTIONS; specified in @storage_sql_dsn
+--
+--   MySQL note ( http://dev.mysql.com/doc/mysql/en/storage-engines.html ):
+--     ENGINE is the preferred term, but cannot be used before MySQL 4.0.18.
+--     TYPE is available beginning with MySQL 3.23.0, the first version of
+--     MySQL for which multiple storage engines were available. If you omit
+--     the ENGINE or TYPE option, the default storage engine is used.
+--     By default this is MyISAM.
+--
+--  Please create additional indexes on keys when needed, or drop suggested
+--  ones as appropriate to optimize queries needed by a management application.
+--  See your database documentation for further optimization hints. With MySQL
+--  see Chapter 15 of the reference manual. For example the chapter 15.17 says:
+--  InnoDB does not keep an internal count of rows in a table. To process a
+--  SELECT COUNT(*) FROM T statement, InnoDB must scan an index of the table,
+--  which takes some time if the index is not entirely in the buffer pool.
+--
+--        Wayne Smith adds: When using MySQL with InnoDB one might want to
+--  increase buffer size for both pool and log, and might also want
+--  to change flush settings for a little better performance. Example:
+--    innodb_buffer_pool_size = 384M
+--    innodb_log_buffer_size = 8M
+--    innodb_flush_log_at_trx_commit = 0
+--  The big performance increase is the first two, the third just helps with
+--  lowering disk activity. Consider also adjusting the key_buffer_size.
+
+-- provide unique id for each e-mail address, avoids storing copies
+CREATE TABLE maddr (
+  partition_tag integer      DEFAULT 0, -- see $partition_tag
+  id         bigint unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  email      varbinary(255)  NOT NULL,  -- full mail address
+  domain     varchar(255)    NOT NULL,  -- only domain part of the email address
+                                        -- with subdomain fields in reverse
+  CONSTRAINT part_email UNIQUE (partition_tag,email)
+) ENGINE=InnoDB;
+
+-- information pertaining to each processed message as a whole;
+-- NOTE: records with NULL msgs.content should be ignored by utilities,
+--   as such records correspond to messages just being processes, or were lost
+-- NOTE: instead of a character field time_iso, one might prefer:
+--   time_iso TIMESTAMP NOT NULL DEFAULT 0,
+--   but the following MUST then be set in amavisd.conf: $timestamp_fmt_mysql=1
+CREATE TABLE msgs (
+  partition_tag integer     DEFAULT 0,   -- see $partition_tag
+  mail_id     varbinary(16) NOT NULL,    -- long-term unique mail id, dflt 12 ch
+  secret_id   varbinary(16) DEFAULT '',  -- authorizes release of mail_id, 12 ch
+  am_id       varchar(20)   NOT NULL,    -- id used in the log
+  time_num    integer unsigned NOT NULL, -- rx_time: seconds since Unix epoch
+  time_iso    char(16)      NOT NULL,    -- rx_time: ISO8601 UTC ascii time
+  sid         bigint unsigned NOT NULL,  -- sender: maddr.id
+  policy      varchar(255)  DEFAULT '',  -- policy bank path (like macro %p)
+  client_addr varchar(255)  DEFAULT '',  -- SMTP client IP address (IPv4 or v6)
+  size        integer unsigned NOT NULL, -- message size in bytes
+  originating char(1) DEFAULT ' ' NOT NULL,  -- sender from inside or auth'd
+  content     char(1),                   -- content type: V/B/U/S/Y/M/H/O/T/C
+    -- virus/banned/unchecked/spam(kill)/spammy(tag2)/
+    -- /bad-mime/bad-header/oversized/mta-err/clean
+    -- is NULL on partially processed mail
+    -- (prior to 2.7.0 the CC_SPAMMY was logged as 's', now 'Y' is used;
+    -- to avoid a need for case-insenstivity in queries)
+  quar_type  char(1),                   -- quarantined as: ' '/F/Z/B/Q/M/L
+                                        --  none/file/zipfile/bsmtp/sql/
+                                        --  /mailbox(smtp)/mailbox(lmtp)
+  quar_loc   varbinary(255) DEFAULT '', -- quarantine location (e.g. file)
+  dsn_sent   char(1),                   -- was DSN sent? Y/N/q (q=quenched)
+  spam_level float,                     -- SA spam level (no boosts)
+  message_id varchar(255)  DEFAULT '',  -- mail Message-ID header field
+  from_addr  varchar(255)  CHARACTER SET utf8mb4 COLLATE utf8mb4_bin  DEFAULT '',
+                                        -- mail From header field,    UTF8
+  subject    varchar(255)  CHARACTER SET utf8mb4 COLLATE utf8mb4_bin  DEFAULT '',
+                                        -- mail Subject header field, UTF8
+  host       varchar(255)  NOT NULL,    -- hostname where amavisd is running
+  PRIMARY KEY (partition_tag,mail_id),
+  FOREIGN KEY (sid) REFERENCES maddr(id) ON DELETE RESTRICT
+) ENGINE=InnoDB;
+CREATE INDEX msgs_idx_sid      ON msgs (sid);
+CREATE INDEX msgs_idx_mess_id  ON msgs (message_id); -- useful with pen pals
+CREATE INDEX msgs_idx_time_num ON msgs (time_num);
+-- alternatively when purging based on time_iso (instead of msgs_idx_time_num):
+CREATE INDEX msgs_idx_time_iso ON msgs (time_iso);
+-- When using FOREIGN KEY contraints, InnoDB requires index on a field
+-- (an the field must be the first field in the index).  Hence create it:
+CREATE INDEX msgs_idx_mail_id  ON msgs (mail_id);
+
+-- per-recipient information related to each processed message;
+-- NOTE: records in msgrcpt without corresponding msgs.mail_id record are
+--  orphaned and should be ignored and eventually deleted by external utilities
+CREATE TABLE msgrcpt (
+  partition_tag integer    DEFAULT 0,    -- see $partition_tag
+  mail_id    varbinary(16) NOT NULL,     -- (must allow duplicates)
+  rseqnum    integer  DEFAULT 0   NOT NULL, -- recip's enumeration within msg
+  rid        bigint unsigned NOT NULL,   -- recipient: maddr.id (dupl. allowed)
+  is_local   char(1)  DEFAULT ' ' NOT NULL, -- recip is: Y=local, N=foreign
+  content    char(1)  DEFAULT ' ' NOT NULL, -- content type V/B/U/S/Y/M/H/O/T/C
+  ds         char(1)  NOT NULL,          -- delivery status: P/R/B/D/T
+                                         -- pass/reject/bounce/discard/tempfail
+  rs         char(1)  NOT NULL,          -- release status: initialized to ' '
+  bl         char(1)  DEFAULT ' ',       -- sender blacklisted by this recip
+  wl         char(1)  DEFAULT ' ',       -- sender whitelisted by this recip
+  bspam_level float,                     -- per-recipient (total) spam level
+  smtp_resp  varchar(255)  DEFAULT '',   -- SMTP response given to MTA
+  PRIMARY KEY (partition_tag,mail_id,rseqnum),
+  FOREIGN KEY (rid)     REFERENCES maddr(id)     ON DELETE RESTRICT,
+  FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE
+) ENGINE=InnoDB;
+CREATE INDEX msgrcpt_idx_mail_id  ON msgrcpt (mail_id);
+CREATE INDEX msgrcpt_idx_rid      ON msgrcpt (rid);
+-- Additional index on rs since Modoboa uses it to filter its quarantine
+CREATE INDEX msgrcpt_idx_rs       ON msgrcpt (rs);
+
+-- mail quarantine in SQL, enabled by $*_quarantine_method='sql:'
+-- NOTE: records in quarantine without corresponding msgs.mail_id record are
+--  orphaned and should be ignored and eventually deleted by external utilities
+CREATE TABLE quarantine (
+  partition_tag integer    DEFAULT 0,    -- see $partition_tag
+  mail_id    varbinary(16) NOT NULL,     -- long-term unique mail id
+  chunk_ind  integer unsigned NOT NULL,  -- chunk number, starting with 1
+  mail_text  blob          NOT NULL,     -- store mail as chunks of octets
+  PRIMARY KEY (partition_tag,mail_id,chunk_ind),
+  FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE
+) ENGINE=InnoDB;

modoboa_installer/scripts/files/amavis/amavis_postgres_2.13.X.sql

@@ -0,0 +1,189 @@
+CREATE TABLE policy (
+  id            serial PRIMARY KEY, -- 'id' is the _only_ required field
+  policy_name   varchar(32),        -- not used by amavisd-new, a comment
+
+  virus_lover           char(1) default NULL,     -- Y/N
+  spam_lover            char(1) default NULL,     -- Y/N
+  unchecked_lover       char(1) default NULL,     -- Y/N
+  banned_files_lover    char(1) default NULL,     -- Y/N
+  bad_header_lover      char(1) default NULL,     -- Y/N
+
+  bypass_virus_checks   char(1) default NULL,     -- Y/N
+  bypass_spam_checks    char(1) default NULL,     -- Y/N
+  bypass_banned_checks  char(1) default NULL,     -- Y/N
+  bypass_header_checks  char(1) default NULL,     -- Y/N
+
+  virus_quarantine_to      varchar(64) default NULL,
+  spam_quarantine_to       varchar(64) default NULL,
+  banned_quarantine_to     varchar(64) default NULL,
+  unchecked_quarantine_to  varchar(64) default NULL,
+  bad_header_quarantine_to varchar(64) default NULL,
+  clean_quarantine_to      varchar(64) default NULL,
+  archive_quarantine_to    varchar(64) default NULL,
+
+  spam_tag_level  real default NULL, -- higher score inserts spam info headers
+  spam_tag2_level real default NULL, -- inserts 'declared spam' header fields
+  spam_tag3_level real default NULL, -- inserts 'blatant spam' header fields
+  spam_kill_level real default NULL, -- higher score triggers evasive actions
+                                     -- e.g. reject/drop, quarantine, ...
+                                     -- (subject to final_spam_destiny setting)
+
+  spam_dsn_cutoff_level        real default NULL,
+  spam_quarantine_cutoff_level real default NULL,
+
+  addr_extension_virus      varchar(64) default NULL,
+  addr_extension_spam       varchar(64) default NULL,
+  addr_extension_banned     varchar(64) default NULL,
+  addr_extension_bad_header varchar(64) default NULL,
+
+  warnvirusrecip      char(1)     default NULL, -- Y/N
+  warnbannedrecip     char(1)     default NULL, -- Y/N
+  warnbadhrecip       char(1)     default NULL, -- Y/N
+  newvirus_admin      varchar(64) default NULL,
+  virus_admin         varchar(64) default NULL,
+  banned_admin        varchar(64) default NULL,
+  bad_header_admin    varchar(64) default NULL,
+  spam_admin          varchar(64) default NULL,
+  spam_subject_tag    varchar(64) default NULL,
+  spam_subject_tag2   varchar(64) default NULL,
+  spam_subject_tag3   varchar(64) default NULL,
+  message_size_limit  integer     default NULL, -- max size in bytes, 0 disable
+  banned_rulenames    varchar(64) default NULL, -- comma-separated list of ...
+        -- names mapped through %banned_rules to actual banned_filename tables
+  disclaimer_options  varchar(64) default NULL,
+  forward_method      varchar(64) default NULL,
+  sa_userconf         varchar(64) default NULL,
+  sa_username         varchar(64) default NULL
+);
+
+-- local users
+CREATE TABLE users (
+  id         serial  PRIMARY KEY,  -- unique id
+  priority   integer NOT NULL DEFAULT 7,  -- sort field, 0 is low prior.
+  policy_id  integer NOT NULL DEFAULT 1 CHECK (policy_id >= 0) REFERENCES policy(id),
+  email      bytea   NOT NULL UNIQUE,     -- email address, non-rfc2822-quoted
+  fullname   varchar(255) DEFAULT NULL    -- not used by amavisd-new
+  -- local   char(1)      -- Y/N  (optional, see SQL section in README.lookups)
+);
+
+-- any e-mail address (non- rfc2822-quoted), external or local,
+-- used as senders in wblist
+CREATE TABLE mailaddr (
+  id         serial  PRIMARY KEY,
+  priority   integer NOT NULL DEFAULT 9,  -- 0 is low priority
+  email      bytea   NOT NULL UNIQUE
+);
+
+-- per-recipient whitelist and/or blacklist,
+-- puts sender and recipient in relation wb  (white or blacklisted sender)
+CREATE TABLE wblist (
+  rid        integer NOT NULL CHECK (rid >= 0) REFERENCES users(id),
+  sid        integer NOT NULL CHECK (sid >= 0) REFERENCES mailaddr(id),
+  wb         varchar(10) NOT NULL,  -- W or Y / B or N / space=neutral / score
+  PRIMARY KEY (rid,sid)
+);
+
+-- grant usage rights:
+GRANT select ON policy   TO amavis;
+GRANT select ON users    TO amavis;
+GRANT select ON mailaddr TO amavis;
+GRANT select ON wblist   TO amavis;
+
+
+-- R/W part of the dataset (optional)
+--   May reside in the same or in a separate database as lookups database;
+--   REQUIRES SUPPORT FOR TRANSACTIONS; specified in @storage_sql_dsn
+--
+--  Please create additional indexes on keys when needed, or drop suggested
+--  ones as appropriate to optimize queries needed by a management application.
+--  See your database documentation for further optimization hints.
+
+-- provide unique id for each e-mail address, avoids storing copies
+CREATE TABLE maddr (
+  id         serial       PRIMARY KEY,
+  partition_tag integer   DEFAULT 0,   -- see $partition_tag
+  email      bytea        NOT NULL,    -- full e-mail address
+  domain     varchar(255) NOT NULL,    -- only domain part of the email address
+                                       -- with subdomain fields in reverse
+  CONSTRAINT part_email UNIQUE (partition_tag,email)
+);
+
+-- information pertaining to each processed message as a whole;
+-- NOTE: records with a NULL msgs.content should be ignored by utilities,
+--   as such records correspond to messages just being processed, or were lost
+CREATE TABLE msgs (
+  partition_tag integer     DEFAULT 0,  -- see $partition_tag
+  mail_id     bytea         NOT NULL,   -- long-term unique mail id, dflt 12 ch
+  secret_id   bytea         DEFAULT '', -- authorizes release of mail_id, 12 ch
+  am_id       varchar(20)   NOT NULL,   -- id used in the log
+  time_num    integer NOT NULL CHECK (time_num >= 0),
+                                        -- rx_time: seconds since Unix epoch
+  time_iso timestamp WITH TIME ZONE NOT NULL,-- rx_time: ISO8601 UTC ascii time
+  sid         integer NOT NULL CHECK (sid >= 0), -- sender: maddr.id
+  policy      varchar(255)  DEFAULT '', -- policy bank path (like macro %p)
+  client_addr varchar(255)  DEFAULT '', -- SMTP client IP address (IPv4 or v6)
+  size        integer NOT NULL CHECK (size >= 0), -- message size in bytes
+  originating char(1) DEFAULT ' ' NOT NULL,  -- sender from inside or auth'd
+  content     char(1),                   -- content type: V/B/U/S/Y/M/H/O/T/C
+    -- virus/banned/unchecked/spam(kill)/spammy(tag2)/
+    -- /bad-mime/bad-header/oversized/mta-err/clean
+    -- is NULL on partially processed mail
+    -- (prior to 2.7.0 the CC_SPAMMY was logged as 's', now 'Y' is used;
+    --- to avoid a need for case-insenstivity in queries)
+  quar_type  char(1),                   -- quarantined as: ' '/F/Z/B/Q/M/L
+                                        --  none/file/zipfile/bsmtp/sql/
+                                        --  /mailbox(smtp)/mailbox(lmtp)
+  quar_loc   varchar(255)  DEFAULT '',  -- quarantine location (e.g. file)
+  dsn_sent   char(1),                   -- was DSN sent? Y/N/q (q=quenched)
+  spam_level real,                      -- SA spam level (no boosts)
+  message_id varchar(255)  DEFAULT '',  -- mail Message-ID header field
+  from_addr  varchar(255)  DEFAULT '',  -- mail From header field,    UTF8
+  subject    varchar(255)  DEFAULT '',  -- mail Subject header field, UTF8
+  host       varchar(255)  NOT NULL,    -- hostname where amavisd is running
+  CONSTRAINT msgs_partition_mail UNIQUE (partition_tag,mail_id),
+  PRIMARY KEY (partition_tag,mail_id)
+--FOREIGN KEY (sid) REFERENCES maddr(id) ON DELETE RESTRICT
+);
+CREATE INDEX msgs_idx_sid      ON msgs (sid);
+CREATE INDEX msgs_idx_mess_id  ON msgs (message_id); -- useful with pen pals
+CREATE INDEX msgs_idx_time_iso ON msgs (time_iso);
+CREATE INDEX msgs_idx_time_num ON msgs (time_num);   -- optional
+
+-- per-recipient information related to each processed message;
+-- NOTE: records in msgrcpt without corresponding msgs.mail_id record are
+--  orphaned and should be ignored and eventually deleted by external utilities
+CREATE TABLE msgrcpt (
+  partition_tag integer DEFAULT 0,  -- see $partition_tag
+  mail_id    bytea    NOT NULL,     -- (must allow duplicates)
+  rseqnum    integer  DEFAULT 0   NOT NULL, -- recip's enumeration within msg
+  rid        integer  NOT NULL,     -- recipient: maddr.id (duplicates allowed)
+  is_local   char(1)  DEFAULT ' ' NOT NULL, -- recip is: Y=local, N=foreign
+  content    char(1)  DEFAULT ' ' NOT NULL, -- content type V/B/U/S/Y/M/H/O/T/C
+  ds         char(1)  NOT NULL,     -- delivery status: P/R/B/D/T
+                                    -- pass/reject/bounce/discard/tempfail
+  rs         char(1)  NOT NULL,     -- release status: initialized to ' '
+  bl         char(1)  DEFAULT ' ',  -- sender blacklisted by this recip
+  wl         char(1)  DEFAULT ' ',  -- sender whitelisted by this recip
+  bspam_level real,                 -- per-recipient (total) spam level
+  smtp_resp  varchar(255) DEFAULT '', -- SMTP response given to MTA
+  CONSTRAINT msgrcpt_partition_mail_rseq UNIQUE (partition_tag,mail_id,rseqnum),
+  PRIMARY KEY (partition_tag,mail_id,rseqnum)
+--FOREIGN KEY (rid)     REFERENCES maddr(id)     ON DELETE RESTRICT,
+--FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE
+);
+CREATE INDEX msgrcpt_idx_mail_id  ON msgrcpt (mail_id);
+CREATE INDEX msgrcpt_idx_rid      ON msgrcpt (rid);
+-- Additional index on rs since Modoboa uses it to filter its quarantine
+CREATE INDEX msgrcpt_idx_rs       ON msgrcpt (rs);
+
+-- mail quarantine in SQL, enabled by $*_quarantine_method='sql:'
+-- NOTE: records in quarantine without corresponding msgs.mail_id record are
+--  orphaned and should be ignored and eventually deleted by external utilities
+CREATE TABLE quarantine (
+  partition_tag integer  DEFAULT 0,      -- see $partition_tag
+  mail_id    bytea   NOT NULL,           -- long-term unique mail id
+  chunk_ind  integer NOT NULL CHECK (chunk_ind >= 0), -- chunk number, 1..
+  mail_text  bytea   NOT NULL,           -- store mail as chunks of octects
+  PRIMARY KEY (partition_tag,mail_id,chunk_ind)
+--FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE
+);

modoboa_installer/scripts/files/automx/automx.conf.tpl

@@ -2,6 +2,9 @@
 provider = %domain
 domains = *
 
+#debug=yes
+#logfile = /srv/automx/automx.log
+
 # Protect against DoS
 memcache = 127.0.0.1:11211
 memcache_ttl = 600
@@ -16,6 +19,8 @@ host = %sql_dsn
 query = %sql_query
 result_attrs = display_name, email
 
+display_name = ${display_name}
+
 smtp = yes
 smtp_server = %hostname
 smtp_port = 587
@@ -32,10 +37,3 @@ imap_encryption = starttls
 imap_auth = plaintext
 imap_auth_identity = ${email}
 imap_refresh_ttl = 6
-
-pop = yes
-pop_server = %hostname
-pop_port = 110
-pop_encryption = starttls
-pop_auth = plaintext
-pop_auth_identity = ${email}

modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf.tpl

@@ -92,14 +92,14 @@ service postlogin {
 service stats {
   # To allow modoboa to access available cipher list.
   unix_listener stats-reader {
-    user = vmail
-    group = vmail
+    user = %{mailboxes_owner}
+    group = %{mailboxes_owner}
     mode = 0660
   }
 
   unix_listener stats-writer {
-    user = vmail
-    group = vmail
+    user = %{mailboxes_owner}
+    group = %{mailboxes_owner}
     mode = 0660
   }
 }
@@ -120,7 +120,7 @@ service auth {
   # permissions (e.g. 0777 allows everyone full permissions).
   unix_listener auth-userdb {
     #mode = 0666
-    user = vmail
+    user = %{mailboxes_owner}
     #group = 
   }
 
@@ -154,7 +154,7 @@ service dict {
   # For example: mode=0660, group=vmail and global mail_access_groups=vmail
   unix_listener dict {
     mode = 0600
-    user = vmail
+    user = %{mailboxes_owner}
     #group = 
   }
 }

modoboa_installer/scripts/files/dovecot/conf.d/10-ssl-keys.try.tpl

@@ -0,0 +1,6 @@
+# PEM encoded X.509 SSL/TLS certificate and private key. They're opened before
+# dropping root privileges, so keep the key file unreadable by anyone but
+# root. Included doc/mkcert.sh can be used to easily generate self-signed
+# certificate, just make sure to update the domains in dovecot-openssl.cnf
+ssl_cert = <%tls_cert_file
+ssl_key = <%tls_key_file

modoboa_installer/scripts/files/dovecot/conf.d/10-ssl.conf.tpl

@@ -5,12 +5,11 @@
 # SSL/TLS support: yes, no, required. <doc/wiki/SSL.txt>
 #ssl = yes
 
-# PEM encoded X.509 SSL/TLS certificate and private key. They're opened before
-# dropping root privileges, so keep the key file unreadable by anyone but
-# root. Included doc/mkcert.sh can be used to easily generate self-signed
-# certificate, just make sure to update the domains in dovecot-openssl.cnf
-ssl_cert = <%tls_cert_file
-ssl_key = <%tls_key_file
+# Workarround https://github.com/modoboa/modoboa/issues/2570 
+# We try to load the key and pass if it fails
+# Keys require root permissions, standard commands would be blocked
+# because dovecot can't load these cert
+!include_try /etc/dovecot/conf.d/10-ssl-keys.try
 
 # If key file is password protected, give the password here. Alternatively
 # give it when starting dovecot with -p parameter. Since this file is often
@@ -41,7 +40,7 @@ ssl_key = <%tls_key_file
 #ssl_parameters_regenerate = 168
 
 # SSL protocols to use
-ssl_protocols = %ssl_protocols
+%ssl_protocol_parameter = %ssl_protocols
 
 
 # SSL ciphers to use

modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl

@@ -123,7 +123,8 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser
 #user_query = \
 #  SELECT home, uid, gid \
 #  FROM users WHERE username = '%%n' AND domain = '%%d'
-user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d'
+%{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d'
+%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only=0 OR '%%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d'
 
 # If you wish to avoid two SQL lookups (passdb + userdb), you can use
 # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll
@@ -133,7 +134,8 @@ user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid,
 #  SELECT userid AS user, password, \
 #    home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \
 #  FROM users WHERE userid = '%%u'
-password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE u.email='%%u' AND u.is_active=1 AND dom.enabled=1
+%{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE u.email='%%u' AND u.is_active=1 AND dom.enabled=1
+%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only=0 OR '%%s' NOT IN ('imap', 'pop3')) AND u.email='%%u' AND u.is_active=1 AND dom.enabled=1
 
 # Query to get a list of all usernames.
 #iterate_query = SELECT username AS user FROM users

modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl

@@ -123,7 +123,8 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser
 #user_query = \
 #  SELECT home, uid, gid \
 #  FROM users WHERE username = '%%n' AND domain = '%%d'
-user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d'
+%{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d'
+%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only IS NOT TRUE OR '%%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d'
 
 # If you wish to avoid two SQL lookups (passdb + userdb), you can use
 # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll
@@ -133,7 +134,8 @@ user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid,
 #  SELECT userid AS user, password, \
 #    home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \
 #  FROM users WHERE userid = '%%u'
-password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE email='%%u' AND is_active AND dom.enabled
+%{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE email='%%u' AND is_active AND dom.enabled
+%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only IS NOT TRUE OR '%%s' NOT IN ('imap', 'pop3')) AND email='%%u' AND is_active AND dom.enabled
 
 # Query to get a list of all usernames.
 #iterate_query = SELECT username AS user FROM users

modoboa_installer/scripts/files/fail2ban/filter.d/modoboa-auth.conf.tpl

@@ -0,0 +1,9 @@
+# Fail2Ban filter Modoboa authentication
+
+[INCLUDES]
+
+before = common.conf
+
+[Definition]
+
+failregex = modoboa\.auth: WARNING Failed connection attempt from \'<HOST>\' as user \'.*?\'$

modoboa_installer/scripts/files/fail2ban/jail.d/modoboa.conf.tpl

@@ -0,0 +1,9 @@
+[modoboa]
+enabled = true
+port = http,https
+protocol = tcp
+filter = modoboa-auth
+maxretry = %max_retry
+bantime = %ban_time
+findtime = %find_time
+logpath = /var/log/auth.log

modoboa_installer/scripts/files/modoboa/crontab.tpl

@@ -30,7 +30,7 @@ INSTANCE=%{instance_path}
 */30    *       *       *       *       root    $PYTHON $INSTANCE/manage.py modo check_mx
 
 # Public API communication
-0       *       *       *       *       root    $PYTHON $INSTANCE/manage.py communicate_with_public_api
+%{minutes}       %{hours}       *       *       *       root    $PYTHON $INSTANCE/manage.py communicate_with_public_api
 
 # Generate DKIM keys (they will belong to the user running this job)
-%{opendkim_enabled}*       *       *       *       *       %{opendkim_user}    umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys
+%{dkim_cron_enabled}*       *       *       *       *       %{opendkim_user}    umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys

modoboa_installer/scripts/files/modoboa/supervisor-rq-base.tpl

@@ -0,0 +1,9 @@
+[program:modoboa-base-worker]
+autostart=true
+autorestart=true
+command=%{venv_path}/bin/python %{home_dir}/instance/manage.py rqworker modoboa
+directory=%{home_dir}
+user=%{user}
+redirect_stderr=true
+numprocs=1
+stopsignal=TERM

modoboa_installer/scripts/files/modoboa/supervisor-rq-dkim.tpl

@@ -0,0 +1,9 @@
+[program:modoboa-dkim-worker]
+autostart=true
+autorestart=true
+command=%{venv_path}/bin/python %{home_dir}/instance/manage.py rqworker dkim
+directory=%{home_dir}
+user=%{opendkim_user}
+redirect_stderr=true
+numprocs=1
+stopsignal=TERM

modoboa_installer/scripts/files/modoboa/supervisor.tpl

@@ -6,3 +6,4 @@ directory=%{home_dir}
 redirect_stderr=true
 user=%{user}
 numprocs=1
+

modoboa_installer/scripts/files/nginx/modoboa.conf.tpl

@@ -10,8 +10,8 @@ server {
 }
 
 server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
     server_name %hostname;
     root %app_instance_path;
 

modoboa_installer/scripts/files/postfix/main.cf.tpl

@@ -41,7 +41,7 @@ smtpd_tls_auth_only = no
 smtpd_tls_CApath = /etc/ssl/certs
 smtpd_tls_key_file = %tls_key_file
 smtpd_tls_cert_file = %tls_cert_file
-smtpd_tls_dh1024_param_file = ${config_directory}/dh2048.pem
+smtpd_tls_dh1024_param_file = ${config_directory}/ffdhe%{dhe_group}.pem
 smtpd_tls_loglevel = 1
 smtpd_tls_session_cache_database = btree:$data_directory/smtpd_tls_session_cache
 smtpd_tls_security_level = may
@@ -57,6 +57,11 @@ smtpd_tls_exclude_ciphers = aNULL, MD5 , DES, ADH, RC4, PSD, SRP, 3DES, eNULL
 # Enable elliptic curve cryptography
 smtpd_tls_eecdh_grade = strong
 
+# SMTP Smuggling prevention
+# See https://www.postfix.org/smtp-smuggling.html
+smtpd_data_restrictions = reject_unauth_pipelining
+smtpd_forbid_unauth_pipelining = yes
+
 # Use TLS if this is supported by the remote SMTP server, otherwise use plaintext.
 smtp_tls_CApath = /etc/ssl/certs
 smtp_tls_security_level = may
@@ -67,10 +72,10 @@ smtp_tls_exclude_ciphers = EXPORT, LOW
 #
 %{dovecot_enabled}virtual_transport = lmtp:unix:private/dovecot-lmtp
 
-virtual_mailbox_domains = proxy:%{db_driver}:/etc/postfix/sql-domains.cf
-virtual_alias_domains = proxy:%{db_driver}:/etc/postfix/sql-domain-aliases.cf
-virtual_alias_maps =
-        proxy:%{db_driver}:/etc/postfix/sql-aliases.cf
+%{dovecot_enabled}virtual_mailbox_domains = proxy:%{db_driver}:/etc/postfix/sql-domains.cf
+%{dovecot_enabled}virtual_alias_domains = proxy:%{db_driver}:/etc/postfix/sql-domain-aliases.cf
+%{dovecot_enabled}virtual_alias_maps =
+%{dovecot_enabled}        proxy:%{db_driver}:/etc/postfix/sql-aliases.cf
 
 ## Relay domains
 #

modoboa_installer/scripts/files/postfix/master.cf.tpl

@@ -78,7 +78,7 @@ scache    unix  -       -       -       -       1       scache
 # Also specify in main.cf: maildrop_destination_recipient_limit=1
 #
 maildrop  unix  -       n       n       -       -       pipe
-  flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient}
+  flags=DRhu user=%{dovecot_mailboxes_owner} argv=/usr/bin/maildrop -d ${recipient}
 #
 # ====================================================================
 #

modoboa_installer/scripts/modoboa.py

@@ -3,6 +3,7 @@
 import json
 import os
 import pwd
+import random
 import shutil
 import stat
 import sys
@@ -25,7 +26,8 @@ class Modoboa(base.Installer):
         "deb": [
             "build-essential", "python3-dev", "libxml2-dev", "libxslt-dev",
             "libjpeg-dev", "librrd-dev", "rrdtool", "libffi-dev", "cron",
-            "libssl-dev", "redis-server", "supervisor"
+            "libssl-dev", "redis-server", "supervisor", "pkg-config",
+            "libcairo2-dev"
         ],
         "rpm": [
             "gcc", "gcc-c++", "python3-devel", "libxml2-devel", "libxslt-devel",
@@ -42,7 +44,7 @@ class Modoboa(base.Installer):
 
     def __init__(self, *args, **kwargs):
         """Get configuration."""
-        super(Modoboa, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.venv_path = self.config.get("modoboa", "venv_path")
         self.instance_path = self.config.get("modoboa", "instance_path")
         self.extensions = self.config.get("modoboa", "extensions").split()
@@ -59,6 +61,7 @@ class Modoboa(base.Installer):
                 self.extensions.remove("modoboa-radicale")
         self.dovecot_enabled = self.config.getboolean("dovecot", "enabled")
         self.opendkim_enabled = self.config.getboolean("opendkim", "enabled")
+        self.dkim_cron_enabled = False
 
     def is_extension_ok_for_version(self, extension, version):
         """Check if extension can be installed with this modo version."""
@@ -86,36 +89,41 @@ class Modoboa(base.Installer):
                     continue
                 if extension in matrix:
                     req_version = matrix[extension]
+                    if req_version is None:
+                        continue
                     req_version = req_version.replace("<", "\<")
                     req_version = req_version.replace(">", "\>")
                     packages.append("{}{}".format(extension, req_version))
                 else:
                     packages.append(extension)
-        # Temp fix for django-braces
-        python.install_package(
-            "django-braces", self.venv_path, upgrade=self.upgrade,
-            sudo_user=self.user
-        )
-        if self.dbengine == "postgres":
-            packages.append("psycopg2-binary\<2.9")
-        else:
-            packages.append("mysqlclient")
         if sys.version_info.major == 2 and sys.version_info.micro < 9:
             # Add extra packages to fix the SNI issue
             packages += ["pyOpenSSL"]
-        # Temp fix for https://github.com/modoboa/modoboa/issues/2247
-        packages.append("django-webpack-loader==0.7.0")
         python.install_packages(
             packages, self.venv_path,
             upgrade=self.upgrade,
             sudo_user=self.user,
             beta=self.config.getboolean("modoboa", "install_beta")
         )
+
+        # Install version specific modules to the venv
+        modoboa_version = ".".join(str(i) for i in python.get_package_version(
+            "modoboa", self.venv_path, sudo_user=self.user
+        ))
+        # Database:
+        db_file = "postgresql"
+        if self.dbengine != "postgres":
+            db_file = "mysql"
+        db_file += "-requirements.txt"
+
+        python.install_package_from_remote_requirements(
+            f"https://raw.githubusercontent.com/modoboa/modoboa/{modoboa_version}/{db_file}",
+            venv=self.venv_path)
+        # Dev mode:
         if self.devmode:
-            # FIXME: use dev-requirements instead
-            python.install_packages(
-                ["django-bower", "django-debug-toolbar"], self.venv_path,
-                upgrade=self.upgrade, sudo_user=self.user)
+            python.install_package_from_remote_requirements(
+                f"https://raw.githubusercontent.com/modoboa/modoboa/{modoboa_version}/dev-requirements.txt",
+                venv=self.venv_path)
 
     def _deploy_instance(self):
         """Deploy Modoboa."""
@@ -176,7 +184,7 @@ class Modoboa(base.Installer):
         if self.upgrade and self.opendkim_enabled and self.dbengine == "postgres":
             # Restore view previously deleted
             self.backend.load_sql_file(
-                self.dbname, self.dbuser, self.dbpassword,
+                self.dbname, self.dbuser, self.dbpasswd,
                 self.get_file_path("dkim_view_{}.sql".format(self.dbengine))
             )
             self.backend.grant_right_on_table(
@@ -186,7 +194,7 @@ class Modoboa(base.Installer):
 
     def setup_database(self):
         """Additional config."""
-        super(Modoboa, self).setup_database()
+        super().setup_database()
         if not self.amavis_enabled:
             return
         self.backend.grant_access(
@@ -194,7 +202,7 @@ class Modoboa(base.Installer):
 
     def get_packages(self):
         """Include extra packages if needed."""
-        packages = super(Modoboa, self).get_packages()
+        packages = super().get_packages()
         condition = (
             package.backend.FORMAT == "rpm" and
             sys.version_info.major == 2 and
@@ -204,6 +212,10 @@ class Modoboa(base.Installer):
             packages += ["openssl-devel"]
         return packages
 
+    def setup_user(self):
+        super().setup_user()
+        self._setup_venv()
+
     def get_config_files(self):
         """Return appropriate path."""
         config_files = super().get_config_files()
@@ -212,6 +224,13 @@ class Modoboa(base.Installer):
         else:
             path = "supervisor=/etc/supervisord.d/policyd.ini"
         config_files.append(path)
+
+        # Add worker for dkim if needed
+        if self.modoboa_2_2_or_greater:
+            config_files.append(
+                "supervisor-rq-dkim=/etc/supervisor/conf.d/modoboa-dkim-worker.conf")
+            config_files.append(
+                "supervisor-rq-base=/etc/supervisor/conf.d/modoboa-base-worker.conf")
         return config_files
 
     def get_template_context(self):
@@ -219,6 +238,9 @@ class Modoboa(base.Installer):
         context = super(Modoboa, self).get_template_context()
         extensions = self.config.get("modoboa", "extensions")
         extensions = extensions.split()
+        random_hour = random.randint(0, 6)
+        self.dkim_cron_enabled = (not self.modoboa_2_2_or_greater and
+                                  self.opendkim_enabled)
         context.update({
             "sudo_user": (
                 "uwsgi" if package.backend.FORMAT == "rpm" else context["user"]
@@ -228,6 +250,10 @@ class Modoboa(base.Installer):
             "radicale_enabled": (
                 "" if "modoboa-radicale" in extensions else "#"),
             "opendkim_user": self.config.get("opendkim", "user"),
+            "minutes": random.randint(1, 59),
+            "hours": f"{random_hour},{random_hour+12}",
+            "modoboa_2_2_or_greater": "" if self.modoboa_2_2_or_greater else "#",
+            "dkim_cron_enabled": "" if self.dkim_cron_enabled else "#"
         })
         return context
 
@@ -239,7 +265,7 @@ class Modoboa(base.Installer):
             self.instance_path, "media", "webmail")
         pw = pwd.getpwnam(self.user)
         for d in [rrd_root_dir, pdf_storage_dir, webmail_media_dir]:
-            utils.mkdir(d, stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3])
+            utils.mkdir_safe(d, stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3])
         settings = {
             "admin": {
                 "handle_mailboxes": True,
@@ -251,7 +277,7 @@ class Modoboa(base.Installer):
             "maillog": {
                 "rrd_rootdir": rrd_root_dir,
             },
-            "modoboa_pdfcredentials": {
+            "pdfcredentials": {
                 "storage_dir": pdf_storage_dir
             },
             "modoboa_radicale": {
@@ -277,17 +303,18 @@ class Modoboa(base.Installer):
 
     def post_run(self):
         """Additional tasks."""
-        self._setup_venv()
+        if 'centos' in utils.dist_name():
+            system.enable_and_start_service("redis")
+        else:
+            system.enable_and_start_service("redis-server")
         self._deploy_instance()
         if not self.upgrade:
             self.apply_settings()
 
         if 'centos' in utils.dist_name():
             supervisor = "supervisord"
-            system.enable_and_start_service("redis")
         else:
             supervisor = "supervisor"
-            system.enable_and_start_service("redis-server")
         # Restart supervisor
         system.enable_service(supervisor)
         utils.exec_cmd("service {} stop".format(supervisor))

modoboa_installer/scripts/nginx.py

@@ -26,7 +26,7 @@ class Nginx(base.Installer):
             "app_instance_path": (
                 self.config.get(app, "instance_path")),
             "uwsgi_socket_path": (
-                Uwsgi(self.config, self.upgrade).get_socket_path(app))
+                Uwsgi(self.config, self.upgrade, self.restore).get_socket_path(app))
         })
         return context
 

modoboa_installer/scripts/opendkim.py

@@ -2,6 +2,7 @@
 
 import os
 import pwd
+import shutil
 import stat
 
 from .. import database
@@ -46,7 +47,7 @@ class Opendkim(base.Installer):
                     stat.S_IROTH | stat.S_IXOTH,
                     target[1], target[2]
                 )
-        super(Opendkim, self).install_config_files()
+        super().install_config_files()
 
     def get_template_context(self):
         """Additional variables."""
@@ -109,3 +110,25 @@ class Opendkim(base.Installer):
             "s/^After=(.*)$/After=$1 {}/".format(dbservice))
         utils.exec_cmd(
             "perl -pi -e '{}' /lib/systemd/system/opendkim.service".format(pattern))
+
+    def restore(self):
+        """Restore keys."""
+        dkim_keys_backup = os.path.join(
+            self.archive_path, "custom/dkim")
+        keys_storage_dir = self.app_config["keys_storage_dir"]
+        if os.path.isdir(dkim_keys_backup):
+            for file in os.listdir(dkim_keys_backup):
+                file_path = os.path.join(dkim_keys_backup, file)
+                if os.path.isfile(file_path):
+                    utils.copy_file(file_path, keys_storage_dir)
+            utils.success("DKIM keys restored from backup")
+        # Setup permissions
+        user = self.config.get("opendkim", "user")
+        utils.exec_cmd(f"chown -R {user}:{user} {keys_storage_dir}")
+
+    def custom_backup(self, path):
+        """Backup DKIM keys."""
+        if os.path.isdir(self.app_config["keys_storage_dir"]):
+            shutil.copytree(self.app_config["keys_storage_dir"], os.path.join(path, "dkim"))
+            utils.printcolor(
+                "DKIM keys saved!", utils.GREEN)

modoboa_installer/scripts/postfix.py

@@ -10,11 +10,10 @@ from .. import package
 from .. import utils
 
 from . import base
-from . import install
+from . import backup, install
 
 
 class Postfix(base.Installer):
-
     """Postfix installer."""
 
     appname = "postfix"
@@ -51,7 +50,7 @@ class Postfix(base.Installer):
 
     def get_template_context(self):
         """Additional variables."""
-        context = super(Postfix, self).get_template_context()
+        context = super().get_template_context()
         context.update({
             "db_driver": self.db_driver,
             "dovecot_mailboxes_owner": self.config.get(
@@ -65,6 +64,13 @@ class Postfix(base.Installer):
         })
         return context
 
+    def check_dhe_group_file(self):
+        group = self.config.get(self.appname, "dhe_group")
+        file_name = f"ffdhe{group}.pem"
+        if not os.path.exists(f"{self.config_dir}/{file_name}"):
+            url = f"https://raw.githubusercontent.com/internetstandards/dhe_groups/main/{file_name}"
+            utils.exec_cmd(f"wget {url}", cwd=self.config_dir)
+
     def post_run(self):
         """Additional tasks."""
         venv_path = self.config.get("modoboa", "venv_path")
@@ -86,10 +92,8 @@ class Postfix(base.Installer):
             if not os.path.exists(path):
                 utils.copy_file(os.path.join("/etc", f), path)
 
-        # Generate EDH parameters
-        if not os.path.exists("{}/dh2048.pem".format(self.config_dir)):
-            cmd = "openssl dhparam -dsaparam -out dh2048.pem 2048"
-            utils.exec_cmd(cmd, cwd=self.config_dir)
+        # Generate DHE group
+        self.check_dhe_group_file()
 
         # Generate /etc/aliases.db file to avoid warnings
         aliases_file = "/etc/aliases"
@@ -97,4 +101,8 @@ class Postfix(base.Installer):
             utils.exec_cmd("postalias {}".format(aliases_file))
 
         # Postwhite
-        install("postwhite", self.config, self.upgrade)
+        install("postwhite", self.config, self.upgrade, self.archive_path)
+
+    def backup(self, path):
+        """Launch postwhite backup."""
+        backup("postwhite", self.config, path)

modoboa_installer/scripts/postwhite.py

@@ -45,8 +45,25 @@ class Postwhite(base.Installer):
         """Additionnal tasks."""
         install_dir = "/usr/local/bin"
         self.install_from_archive(SPF_TOOLS_REPOSITORY, install_dir)
-        postw_dir = self.install_from_archive(
+        self.postw_dir = self.install_from_archive(
             POSTWHITE_REPOSITORY, install_dir)
-        utils.copy_file(os.path.join(postw_dir, "postwhite.conf"), "/etc")
-        postw_bin = os.path.join(postw_dir, "postwhite")
-        utils.exec_cmd("{} /etc/postwhite.conf".format(postw_bin))
+        utils.copy_file(
+            os.path.join(self.postw_dir, "postwhite.conf"), self.config_dir)
+        self.postw_bin = os.path.join(self.postw_dir, "postwhite")
+        utils.exec_cmd("{} /etc/postwhite.conf".format(self.postw_bin))
+
+    def custom_backup(self, path):
+        """Backup custom configuration if any."""
+        postswhite_custom = "/etc/postwhite.conf"
+        if os.path.isfile(postswhite_custom):
+            utils.copy_file(postswhite_custom, path)
+            utils.printcolor(
+                "Postwhite configuration saved!", utils.GREEN)
+
+    def restore(self):
+        """Restore config files."""
+        postwhite_backup_configuration = os.path.join(
+            self.archive_path, "custom/postwhite.conf")
+        if os.path.isfile(postwhite_backup_configuration):
+            utils.copy_file(postwhite_backup_configuration, self.config_dir)
+            utils.success("postwhite.conf restored from backup")

modoboa_installer/scripts/radicale.py

@@ -1,6 +1,7 @@
 """Radicale related tasks."""
 
 import os
+import shutil
 import stat
 
 from .. import package
@@ -25,7 +26,7 @@ class Radicale(base.Installer):
 
     def __init__(self, *args, **kwargs):
         """Get configuration."""
-        super(Radicale, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.venv_path = self.config.get("radicale", "venv_path")
 
     def _setup_venv(self):
@@ -70,7 +71,18 @@ class Radicale(base.Installer):
                 stat.S_IROTH | stat.S_IXOTH,
                 0, 0
             )
-        super(Radicale, self).install_config_files()
+        super().install_config_files()
+
+    def restore(self):
+        """Restore collections."""
+        radicale_backup = os.path.join(
+            self.archive_path, "custom/radicale")
+        if os.path.isdir(radicale_backup):
+            restore_target = os.path.join(self.home_dir, "collections")
+            if os.path.isdir(restore_target):
+                shutil.rmtree(restore_target)
+            shutil.copytree(radicale_backup, restore_target)
+            utils.success("Radicale collections restored from backup")
 
     def post_run(self):
         """Additional tasks."""
@@ -81,3 +93,12 @@ class Radicale(base.Installer):
         system.enable_service(daemon_name)
         utils.exec_cmd("service {} stop".format(daemon_name))
         utils.exec_cmd("service {} start".format(daemon_name))
+
+    def custom_backup(self, path):
+        """Backup collections."""
+        radicale_backup = os.path.join(self.config.get(
+            "radicale", "home_dir", fallback="/srv/radicale"), "collections")
+        if os.path.isdir(radicale_backup):
+            shutil.copytree(radicale_backup, os.path.join(
+                path, "radicale"))
+            utils.printcolor("Radicale files saved", utils.GREEN)

modoboa_installer/scripts/restore.py

@@ -0,0 +1,26 @@
+import os
+import sys
+from .. import utils
+
+
+class Restore:
+    def __init__(self, restore):
+        """
+        Restoring pre-check (backup integriety)
+        REQUIRED : modoboa.sql
+        OPTIONAL : mails/, custom/, amavis.sql, spamassassin.sql
+        Only checking required
+        """
+
+        if not os.path.isdir(restore):
+            utils.error(
+                "Provided path is not a directory !")
+            sys.exit(1)
+
+        modoba_sql_file = os.path.join(restore, "databases/modoboa.sql")
+        if not os.path.isfile(modoba_sql_file):
+            utils.error(
+                modoba_sql_file + " not found, please check your backup")
+            sys.exit(1)
+
+        # Everything seems alright here, proceeding...

modoboa_installer/scripts/spamassassin.py

@@ -57,13 +57,7 @@ class Spamassassin(base.Installer):
 
     def post_run(self):
         """Additional tasks."""
-        amavis_user = self.config.get("amavis", "user")
-        pw = pwd.getpwnam(amavis_user)
-        utils.exec_cmd(
-            "pyzor --homedir {} discover".format(pw[5]),
-            sudo_user=amavis_user, login=False
-        )
-        install("razor", self.config, self.upgrade)
+        install("razor", self.config, self.upgrade, self.restore)
         if utils.dist_name() in ["debian", "ubuntu"]:
             utils.exec_cmd(
                 "perl -pi -e 's/^CRON=0/CRON=1/' /etc/cron.daily/spamassassin")

modoboa_installer/ssl.py

@@ -90,7 +90,7 @@ class LetsEncryptCertificate(CertificateBackend):
         elif "centos" in name:
             package.backend.install("certbot")
         else:
-            utils.printcolor("Failed to install certbot, aborting.", utils.RED)
+            utils.printcolor("Failed to install certbot, aborting.")
             sys.exit(1)
         # Nginx plugin certbot
         if (

modoboa_installer/utils.py

@@ -2,10 +2,13 @@
 
 import contextlib
 import datetime
+import getpass
 import glob
 import os
+import pwd
 import random
 import shutil
+import stat
 import string
 import subprocess
 import sys
@@ -39,13 +42,15 @@ def user_input(message):
     return answer
 
 
-def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs):
-    """Execute a shell command.
+def exec_cmd(cmd, sudo_user=None, login=True, **kwargs):
+    """
+    Execute a shell command.
+
     Run a command using the current user. Set :keyword:`sudo_user` if
     you need different privileges.
+
     :param str cmd: the command to execute
     :param str sudo_user: a valid system username
-    :param str pinput: data to send to process's stdin
     :rtype: tuple
     :return: return code, command output
     """
@@ -54,23 +59,21 @@ def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs):
         cmd = "sudo {}-u {} {}".format("-i " if login else "", sudo_user, cmd)
     if "shell" not in kwargs:
         kwargs["shell"] = True
-    if pinput is not None:
-        kwargs["stdin"] = subprocess.PIPE
-    capture_output = False
+    capture_output = True
     if "capture_output" in kwargs:
         capture_output = kwargs.pop("capture_output")
-    elif not ENV.get("debug"):
-        capture_output = True
     if capture_output:
-        kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = None
-    process = subprocess.Popen(cmd, **kwargs)
-    if pinput or capture_output:
-        c_args = [pinput] if pinput is not None else []
-        output = process.communicate(*c_args)[0]
-    else:
-        process.wait()
-    return process.returncode, output
+        kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    kwargs["universal_newlines"] = True
+    output: str = ""
+    with subprocess.Popen(cmd, **kwargs) as process:
+        if capture_output:
+            for line in process.stdout:
+                output += line
+                if ENV.get("debug"):
+                    sys.stdout.write(line)
+
+    return process.returncode, output.encode()
 
 
 def dist_info():
@@ -107,6 +110,13 @@ def mkdir(path, mode, uid, gid):
     os.chown(path, uid, gid)
 
 
+def mkdir_safe(path, mode, uid, gid):
+    """Create a directory. Safe way (-p)"""
+    if not os.path.exists(path):
+        os.makedirs(os.path.abspath(path), mode)
+    mkdir(path, mode, uid, gid)
+
+
 def make_password(length=16):
     """Create a random password."""
     return "".join(
@@ -125,7 +135,6 @@ def settings(**kwargs):
 
 
 class ConfigFileTemplate(string.Template):
-
     """Custom class for configuration files."""
 
     delimiter = "%"
@@ -163,19 +172,32 @@ def copy_from_template(template, dest, context):
         fp.write(ConfigFileTemplate(buf).substitute(context))
 
 
-def check_config_file(dest, interactive=False, upgrade=False):
+def check_config_file(dest, interactive=False, upgrade=False, backup=False, restore=False):
     """Create a new installer config file if needed."""
+    is_present = True
     if os.path.exists(dest):
-        return
+        return is_present, update_config(dest, False)
     if upgrade:
         printcolor(
             "You cannot upgrade an existing installation without a "
             "configuration file.", RED)
         sys.exit(1)
+    elif backup:
+        is_present = False
+        printcolor(
+            "Your configuration file hasn't been found. A new one will be generated. "
+            "Please edit it with correct password for the databases !", RED)
+    elif restore:
+        printcolor(
+            "You cannot restore an existing installation without a "
+            f"configuration file. (file : {dest} has not been found...", RED)
+        sys.exit(1)
+
     printcolor(
         "Configuration file {} not found, creating new one."
         .format(dest), YELLOW)
     gen_config(dest, interactive)
+    return is_present, None
 
 
 def has_colours(stream):
@@ -203,6 +225,16 @@ def printcolor(message, color):
     print(message)
 
 
+def error(message):
+    """Print error message."""
+    printcolor(message, RED)
+
+
+def success(message):
+    """Print success message."""
+    printcolor(message, GREEN)
+
+
 def convert_version_to_int(version):
     """Convert a version string to an integer."""
     number_bits = (8, 8, 16)
@@ -287,8 +319,8 @@ def get_entry_value(entry, interactive):
     return user_value if user_value else default_value
 
 
-def gen_config(dest, interactive=False):
-    """Create config file from dict template"""
+def load_config_template(interactive):
+    """Instantiate a configParser object with the predefined template."""
     tpl_dict = config_dict_template.ConfigDictTemplate
     config = configparser.ConfigParser()
     # only ask about options we need, else still generate default
@@ -304,6 +336,141 @@ def gen_config(dest, interactive=False):
         for config_entry in section["values"]:
             value = get_entry_value(config_entry, interactive_section)
             config.set(section["name"], config_entry["option"], value)
+    return config
+
+
+def update_config(path, apply_update=True):
+    """Update an existing config file."""
+    config = configparser.ConfigParser()
+    with open(path) as fp:
+        config.read_file(fp)
+    new_config = load_config_template(False)
+
+    old_sections = config.sections()
+    new_sections = new_config.sections()
+
+    update = False
+
+    dropped_sections = list(set(old_sections) - set(new_sections))
+    added_sections = list(set(new_sections) - set(old_sections))
+    if len(dropped_sections) > 0 and apply_update:
+        printcolor("Following section(s) will not be ported "
+                   "due to being deleted or renamed: " +
+                   ', '.join(dropped_sections),
+                   RED)
+
+    if len(dropped_sections) + len(added_sections) > 0:
+        update = True
+
+    for section in new_sections:
+        if section in old_sections:
+            new_options = new_config.options(section)
+            old_options = config.options(section)
+
+            dropped_options = list(set(old_options) - set(new_options))
+            added_options = list(set(new_options) - set(old_options))
+            if len(dropped_options) > 0 and apply_update:
+                printcolor(f"Following option(s) from section: {section}, "
+                           "will not be ported due to being "
+                           "deleted or renamed: " +
+                           ', '.join(dropped_options),
+                           RED)
+            if len(dropped_options) + len(added_options) > 0:
+                update = True
+
+            if apply_update:
+                for option in new_options:
+                    if option in old_options:
+                        value = config.get(section, option, raw=True)
+                        if value != new_config.get(section, option, raw=True):
+                            update = True
+                            new_config.set(section, option, value)
+    if apply_update:
+        if update:
+            # Backing up old config file
+            date = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
+            dest = f"{os.path.splitext(path)[0]}_{date}.old"
+            shutil.copy(path, dest)
+
+            # Overwritting old config file
+            with open(path, "w") as configfile:
+                new_config.write(configfile)
+
+            # Set file owner to running u+g, and set config file permission to 600
+            current_username = getpass.getuser()
+            current_user = pwd.getpwnam(current_username)
+            os.chown(dest, current_user[2], current_user[3])
+            os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR)
+
+            return dest
+        return None
+    else:
+        # Simply check if current config file is outdated
+        return update
+
+
+def gen_config(dest, interactive=False):
+    """Create config file from dict template"""
+    config = load_config_template(interactive)
 
     with open(dest, "w") as configfile:
         config.write(configfile)
+
+    # Set file owner to running user and group, and set config file permission to 600
+    current_username = getpass.getuser()
+    current_user = pwd.getpwnam(current_username)
+    os.chown(dest, current_user[2], current_user[3])
+    os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR)
+
+
+def validate_backup_path(path: str, silent_mode: bool):
+    """Check if provided backup path is valid or not."""
+    path_exists = os.path.exists(path)
+    if path_exists and os.path.isfile(path):
+        printcolor(
+            "Error, you provided a file instead of a directory!", RED)
+        return None
+
+    if not path_exists:
+        if not silent_mode:
+            create_dir = input(
+                f"\"{path}\" doesn't exist, would you like to create it? [Y/n]\n"
+            ).lower()
+
+        if silent_mode or (not silent_mode and create_dir.startswith("y")):
+            pw = pwd.getpwnam("root")
+            mkdir_safe(path, stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3])
+        else:
+            printcolor(
+                "Error, backup directory not present.", RED
+            )
+            return None
+
+    if len(os.listdir(path)) != 0:
+        if not silent_mode:
+            delete_dir = input(
+                "Warning: backup directory is not empty, it will be purged if you continue... [Y/n]\n").lower()
+
+        if silent_mode or (not silent_mode and delete_dir.startswith("y")):
+            try:
+                os.remove(os.path.join(path, "installer.cfg"))
+            except FileNotFoundError:
+                pass
+
+            shutil.rmtree(os.path.join(path, "custom"),
+                          ignore_errors=False)
+            shutil.rmtree(os.path.join(path, "mails"), ignore_errors=False)
+            shutil.rmtree(os.path.join(path, "databases"),
+                          ignore_errors=False)
+        else:
+            printcolor(
+                "Error: backup directory not clean.", RED
+            )
+            return None
+
+    backup_path = path
+    pw = pwd.getpwnam("root")
+    for dir in ["custom/", "databases/"]:
+        mkdir_safe(os.path.join(backup_path, dir),
+                   stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3])
+    return backup_path

run.py

@@ -3,6 +3,8 @@
 """An installer for Modoboa."""
 
 import argparse
+import datetime
+import os
 try:
     import configparser
 except ImportError:
@@ -10,6 +12,7 @@ except ImportError:
 import sys
 
 from modoboa_installer import compatibility_matrix
+from modoboa_installer import constants
 from modoboa_installer import package
 from modoboa_installer import scripts
 from modoboa_installer import ssl
@@ -17,6 +20,20 @@ from modoboa_installer import system
 from modoboa_installer import utils
 
 
+PRIMARY_APPS = [
+    "amavis",
+    "fail2ban",
+    "modoboa",
+    "automx",
+    "radicale",
+    "uwsgi",
+    "nginx",
+    "opendkim",
+    "postfix",
+    "dovecot"
+]
+
+
 def installation_disclaimer(args, config):
     """Display installation disclaimer."""
     hostname = config.get("general", "hostname")
@@ -25,7 +42,7 @@ def installation_disclaimer(args, config):
         "Before you start the installation, please make sure the following "
         "DNS records exist for domain '{}':\n"
         "  {} IN A   <IP ADDRESS OF YOUR SERVER>\n"
-        "       IN MX  {}.\n".format(
+        "     @ IN MX  {}.\n".format(
             args.domain,
             hostname.replace(".{}".format(args.domain), ""),
             hostname
@@ -45,6 +62,69 @@ def upgrade_disclaimer(config):
     )
 
 
+def backup_disclaimer():
+    """Display backup disclamer. """
+    utils.printcolor(
+        "Your mail server will be backed up locally.\n"
+        " !! You should really transfer the backup somewhere else...\n"
+        " !! Custom configuration (like for postfix) won't be saved.", utils.BLUE)
+
+
+def restore_disclaimer():
+    """Display restore disclamer. """
+    utils.printcolor(
+        "You are about to restore a previous installation of Modoboa.\n"
+        "If a new version has been released in between, please update your database!",
+        utils.BLUE)
+
+
+def backup_system(config, args):
+    """Launch backup procedure."""
+    backup_disclaimer()
+    backup_path = None
+    if args.silent_backup:
+        if not args.backup_path:
+            if config.has_option("backup", "default_path"):
+                path = config.get("backup", "default_path")
+            else:
+                path = constants.DEFAULT_BACKUP_DIRECTORY
+            date = datetime.datetime.now().strftime("%m_%d_%Y_%H_%M")
+            path = os.path.join(path, f"backup_{date}")
+        else:
+            path = args.backup_path
+        backup_path = utils.validate_backup_path(path, args.silent_backup)
+        if not backup_path:
+            utils.printcolor(f"Path provided: {path}", utils.BLUE)
+            return
+    else:
+        user_value = None
+        while not user_value or not backup_path:
+            utils.printcolor(
+                "Enter backup path (it must be an empty directory)",
+                utils.MAGENTA
+            )
+            utils.printcolor("CTRL+C to cancel", utils.MAGENTA)
+            user_value = utils.user_input("-> ")
+            if not user_value:
+                continue
+            backup_path = utils.validate_backup_path(user_value, args.silent_backup)
+
+    # Backup configuration file
+    utils.copy_file(args.configfile, backup_path)
+    # Backup applications
+    for app in PRIMARY_APPS:
+        scripts.backup(app, config, backup_path)
+
+
+def config_file_update_complete(backup_location):
+    utils.printcolor("Update complete. It seems successful.",
+                     utils.BLUE)
+    if backup_location is not None:
+        utils.printcolor("You will find your old config file "
+                         f"here: {backup_location}",
+                         utils.BLUE)
+
+
 def main(input_args):
     """Install process."""
     parser = argparse.ArgumentParser()
@@ -72,16 +152,68 @@ def main(input_args):
     parser.add_argument(
         "--beta", action="store_true", default=False,
         help="Install latest beta release of Modoboa instead of the stable one")
+    parser.add_argument(
+        "--backup-path", type=str, metavar="path",
+        help="To use with --silent-backup, you must provide a valid path")
+    parser.add_argument(
+        "--backup", action="store_true", default=False,
+        help="Backing up interactively previously installed instance"
+    )
+    parser.add_argument(
+        "--silent-backup", action="store_true", default=False,
+        help="For script usage, do not require user interaction "
+        "backup will be saved at ./modoboa_backup/Backup_M_Y_d_H_M "
+        "if --backup-path is not provided")
+    parser.add_argument(
+        "--restore", type=str, metavar="path",
+        help="Restore a previously backup up modoboa instance on a NEW machine. "
+        "You MUST provide backup directory"
+    )
     parser.add_argument("domain", type=str,
                         help="The main domain of your future mail server")
     args = parser.parse_args(input_args)
 
     if args.debug:
         utils.ENV["debug"] = True
-    utils.printcolor("Welcome to Modoboa installer!\n", utils.GREEN)
-    utils.check_config_file(args.configfile, args.interactive, args.upgrade)
+
+    # Restore prep
+    is_restoring = False
+    if args.restore is not None:
+        is_restoring = True
+        args.configfile = os.path.join(args.restore, args.configfile)
+        if not os.path.exists(args.configfile):
+            utils.error(
+                "Installer configuration file not found in backup!"
+            )
+            sys.exit(1)
+
+    utils.success("Welcome to Modoboa installer!\n")
+
+    is_config_file_available, outdate_config = utils.check_config_file(
+        args.configfile, args.interactive, args.upgrade, args.backup, is_restoring)
+
+    if not is_config_file_available and (
+            args.upgrade or args.backup or args.silent_backup):
+        utils.error("No config file found.")
+        return
+
+    # Check if config is outdated and ask user if it needs to be updated
+    if is_config_file_available and outdate_config:
+        answer = utils.user_input("It seems that your config file is outdated. "
+                                  "Would you like to update it? (Y/n) ")
+        if answer.lower().startswith("y"):
+            config_file_update_complete(utils.update_config(args.configfile))
+            if not args.stop_after_configfile_check:
+                answer = utils.user_input("Would you like to stop to review the updated config? (Y/n)")
+                if answer.lower().startswith("y"):
+                    return
+        else:
+            utils.error("You might encounter unexpected errors ! "
+                        "Make sure to update your config before opening an issue!")
+
     if args.stop_after_configfile_check:
         return
+
     config = configparser.ConfigParser()
     with open(args.configfile) as fp:
         config.read_file(fp)
@@ -91,11 +223,20 @@ def main(input_args):
     config.set("dovecot", "domain", args.domain)
     config.set("modoboa", "version", args.version)
     config.set("modoboa", "install_beta", str(args.beta))
-    # Display disclaimerpython 3 linux distribution
-    if not args.upgrade:
-        installation_disclaimer(args, config)
-    else:
+
+    if args.backup or args.silent_backup:
+        backup_system(config, args)
+        return
+
+    # Display disclaimer python 3 linux distribution
+    if args.upgrade:
         upgrade_disclaimer(config)
+    elif args.restore:
+        restore_disclaimer()
+        scripts.restore_prep(args.restore)
+    else:
+        installation_disclaimer(args, config)
+
     # Show concerned components
     components = []
     for section in config.sections():
@@ -115,27 +256,26 @@ def main(input_args):
     utils.printcolor(
         "The process can be long, feel free to take a coffee "
         "and come back later ;)", utils.BLUE)
-    utils.printcolor("Starting...", utils.GREEN)
+    utils.success("Starting...")
     package.backend.prepare_system()
     package.backend.install_many(["sudo", "wget"])
     ssl_backend = ssl.get_backend(config)
     if ssl_backend and not args.upgrade:
         ssl_backend.generate_cert()
-    scripts.install("amavis", config, args.upgrade)
-    scripts.install("modoboa", config, args.upgrade)
-    scripts.install("automx", config, args.upgrade)
-    scripts.install("radicale", config, args.upgrade)
-    scripts.install("uwsgi", config, args.upgrade)
-    scripts.install("nginx", config, args.upgrade)
-    scripts.install("opendkim", config, args.upgrade)
-    scripts.install("postfix", config, args.upgrade)
-    scripts.install("dovecot", config, args.upgrade)
+    for appname in PRIMARY_APPS:
+        scripts.install(appname, config, args.upgrade, args.restore)
     system.restart_service("cron")
     package.backend.restore_system()
-    utils.printcolor(
-        "Congratulations! You can enjoy Modoboa at https://{} (admin:password)"
-        .format(config.get("general", "hostname")),
-        utils.GREEN)
+    if not args.restore:
+        utils.success(
+            "Congratulations! You can enjoy Modoboa at https://{} (admin:password)"
+            .format(config.get("general", "hostname"))
+        )
+    else:
+        utils.success(
+            "Restore complete! You can enjoy Modoboa at https://{} (same credentials as before)"
+            .format(config.get("general", "hostname"))
+        )
 
 
 if __name__ == "__main__":

test-requirements.txt

@@ -1,3 +1,2 @@
 codecov
 mock
-six

tests.py

@@ -6,8 +6,13 @@ import sys
 import tempfile
 import unittest
 
-from six import StringIO
-from six.moves import configparser
+from io import StringIO
+from pathlib import Path
+
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
 try:
     from unittest.mock import patch
 except ImportError:
@@ -57,6 +62,39 @@ class ConfigFileTestCase(unittest.TestCase):
         self.assertEqual(config.get("certificate", "type"), "self-signed")
         self.assertEqual(config.get("database", "engine"), "postgres")
 
+    @patch("modoboa_installer.utils.user_input")
+    def test_updating_configfile(self, mock_user_input):
+        """Check configfile update mechanism."""
+        cfgfile_temp = os.path.join(self.workdir, "installer_old.cfg")
+
+        out = StringIO()
+        sys.stdout = out
+        run.main([
+            "--stop-after-configfile-check",
+            "--configfile", cfgfile_temp,
+            "example.test"])
+        self.assertTrue(os.path.exists(cfgfile_temp))
+
+        # Adding a dummy section
+        with open(cfgfile_temp, "a") as fp:
+            fp.write(
+"""
+[dummy]
+    weird_old_option = "hey
+""")
+        mock_user_input.side_effect = ["y"]
+        out = StringIO()
+        sys.stdout = out
+        run.main([
+            "--stop-after-configfile-check",
+            "--configfile", cfgfile_temp,
+            "example.test"])
+        self.assertIn("dummy", out.getvalue())
+        self.assertTrue(Path(self.workdir).glob("*.old"))
+        self.assertIn("Update complete",
+                      out.getvalue()
+        )
+
     @patch("modoboa_installer.utils.user_input")
     def test_interactive_mode_letsencrypt(self, mock_user_input):
         """Check interactive mode."""
@@ -92,6 +130,9 @@ class ConfigFileTestCase(unittest.TestCase):
             " postwhite spamassassin uwsgi",
             out.getvalue()
         )
+        self.assertNotIn("It seems that your config file is outdated.",
+                         out.getvalue()
+        )
 
     @patch("modoboa_installer.utils.user_input")
     def test_upgrade_mode(self, mock_user_input):