Merge pull request #514 from modoboa/rq

Updated for 2.2

modoboa_installer/package.py

@@ -46,7 +46,7 @@ class DEBPackage(Package):
         """Update local cache."""
         if self.index_updated:
             return
-        utils.exec_cmd("apt-get update --quiet")
+        utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 update --quiet")
         self.index_updated = True
 
     def preconfigure(self, name, question, qtype, answer):
@@ -57,18 +57,18 @@ class DEBPackage(Package):
     def install(self, name):
         """Install a package."""
         self.update()
-        utils.exec_cmd("apt-get install --quiet --assume-yes {}".format(name))
+        utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 install --quiet --assume-yes {}".format(name))
 
     def install_many(self, names):
         """Install many packages."""
         self.update()
-        return utils.exec_cmd("apt-get install --quiet --assume-yes {}".format(
+        return utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 install --quiet --assume-yes {}".format(
             " ".join(names)))
 
     def get_installed_version(self, name):
         """Get installed package version."""
         code, output = utils.exec_cmd(
-            "dpkg -s {} | grep Version".format(name), capture_output=True)
+            "dpkg -s {} | grep Version".format(name))
         match = re.match(r"Version: (\d:)?(.+)-\d", output.decode())
         if match:
             return match.group(2)
@@ -97,7 +97,7 @@ class RPMPackage(Package):
     def get_installed_version(self, name):
         """Get installed package version."""
         code, output = utils.exec_cmd(
-            "rpm -qi {} | grep Version".format(name), capture_output=True)
+            "rpm -qi {} | grep Version".format(name))
         match = re.match(r"Version\s+: (.+)", output.decode())
         if match:
             return match.group(1)

modoboa_installer/python.py

@@ -1,6 +1,7 @@
 """Python related tools."""
 
 import os
+import sys
 
 from . import package
 from . import utils
@@ -45,6 +46,33 @@ def install_packages(names, venv=None, upgrade=False, **kwargs):
     utils.exec_cmd(cmd, **kwargs)
 
 
+def get_package_version(name, venv=None, **kwargs):
+    """Returns the version of an installed package."""
+    cmd = "{} show {}".format(
+        get_pip_path(venv),
+        name
+    )
+    exit_code, output = utils.exec_cmd(cmd, **kwargs)
+    if exit_code != 0:
+        utils.error(f"Failed to get version of {name}. "
+                    f"Output is: {output}")
+        sys.exit(1)
+
+    output_list = output.decode().split("\n")
+    version_item_list = output_list[1].split(":")
+    version_list = version_item_list[1].split(".")
+    version_list_clean = []
+    for element in version_list:
+        try:
+            version_list_clean.append(int(element))
+        except ValueError:
+            utils.printcolor(
+                f"Failed to decode some part of the version of {name}",
+                utils.YELLOW)
+            version_list_clean.append(element)
+    return version_list_clean
+
+
 def install_package_from_repository(name, url, vcs="git", venv=None, **kwargs):
     """Install a Python package from its repository."""
     if vcs == "git":

modoboa_installer/scripts/base.py

@@ -5,6 +5,7 @@ import sys
 
 from .. import database
 from .. import package
+from .. import python
 from .. import system
 from .. import utils
 
@@ -42,6 +43,20 @@ class Installer(object):
         self.dbuser = self.config.get(self.appname, "dbuser")
         self.dbpasswd = self.config.get(self.appname, "dbpassword")
 
+    @property
+    def modoboa_2_2_or_greater(self):
+        # Check if modoboa version > 2.2
+        modoboa_version = python.get_package_version(
+            "modoboa",
+            self.config.get("modoboa", "venv_path"),
+            sudo_user=self.config.get("modoboa", "user")
+            )
+        condition = (
+            (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or
+            modoboa_version[0] > 2
+            )
+        return condition
+
     @property
     def config_dir(self):
         """Return main configuration directory."""

modoboa_installer/scripts/dovecot.py

@@ -83,6 +83,7 @@ class Dovecot(base.Installer):
         else:
             # Protocols are automatically guessed on debian/ubuntu
             protocols = ""
+
         context.update({
             "db_driver": self.db_driver,
             "mailboxes_owner_uid": pw_mailbox[2],
@@ -97,7 +98,9 @@ class Dovecot(base.Installer):
             "ssl_protocol_parameter": ssl_protocol_parameter,
             "radicale_user": self.config.get("radicale", "user"),
             "radicale_auth_socket_path": os.path.basename(
-                self.config.get("dovecot", "radicale_auth_socket_path"))
+                self.config.get("dovecot", "radicale_auth_socket_path")),
+            "modoboa_2_2_or_greater": "" if self.modoboa_2_2_or_greater else "#",
+            "not_modoboa_2_2_or_greater": "" if not self.modoboa_2_2_or_greater else "#"
         })
         return context
 

modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl

@@ -123,7 +123,8 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser
 #user_query = \
 #  SELECT home, uid, gid \
 #  FROM users WHERE username = '%%n' AND domain = '%%d'
-user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d'
+%{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d'
+%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only=0 OR '%%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d'
 
 # If you wish to avoid two SQL lookups (passdb + userdb), you can use
 # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll
@@ -133,7 +134,8 @@ user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid,
 #  SELECT userid AS user, password, \
 #    home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \
 #  FROM users WHERE userid = '%%u'
-password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE u.email='%%u' AND u.is_active=1 AND dom.enabled=1
+%{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE u.email='%%u' AND u.is_active=1 AND dom.enabled=1
+%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only=0 OR '%%s' NOT IN ('imap', 'pop3')) AND u.email='%%u' AND u.is_active=1 AND dom.enabled=1
 
 # Query to get a list of all usernames.
 #iterate_query = SELECT username AS user FROM users

modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl

@@ -123,7 +123,8 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser
 #user_query = \
 #  SELECT home, uid, gid \
 #  FROM users WHERE username = '%%n' AND domain = '%%d'
-user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d'
+%{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d'
+%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only IS NOT TRUE OR '%%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d'
 
 # If you wish to avoid two SQL lookups (passdb + userdb), you can use
 # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll
@@ -133,7 +134,8 @@ user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid,
 #  SELECT userid AS user, password, \
 #    home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \
 #  FROM users WHERE userid = '%%u'
-password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE email='%%u' AND is_active AND dom.enabled
+%{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE email='%%u' AND is_active AND dom.enabled
+%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only IS NOT TRUE OR '%%s' NOT IN ('imap', 'pop3')) AND email='%%u' AND is_active AND dom.enabled
 
 # Query to get a list of all usernames.
 #iterate_query = SELECT username AS user FROM users

modoboa_installer/scripts/files/modoboa/crontab.tpl

@@ -33,4 +33,4 @@ INSTANCE=%{instance_path}
 %{minutes}       %{hours}       *       *       *       root    $PYTHON $INSTANCE/manage.py communicate_with_public_api
 
 # Generate DKIM keys (they will belong to the user running this job)
-%{opendkim_enabled}*       *       *       *       *       %{opendkim_user}    umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys
+%{dkim_cron_enabled}*       *       *       *       *       %{opendkim_user}    umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys

modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl

@@ -0,0 +1,9 @@
+[program:modoboa-dkim-worker]
+autostart=true
+autorestart=true
+command=%{venv_path}/bin/python %{home_dir}/instance/manage.py rqworker dkim
+directory=%{home_dir}
+user=%{opendkim_user}
+redirect_stderr=true
+numprocs=1
+stopsignal=TERM

modoboa_installer/scripts/files/modoboa/supervisor.tpl

@@ -6,3 +6,4 @@ directory=%{home_dir}
 redirect_stderr=true
 user=%{user}
 numprocs=1
+

modoboa_installer/scripts/modoboa.py

@@ -61,6 +61,7 @@ class Modoboa(base.Installer):
                 self.extensions.remove("modoboa-radicale")
         self.dovecot_enabled = self.config.getboolean("dovecot", "enabled")
         self.opendkim_enabled = self.config.getboolean("opendkim", "enabled")
+        self.dkim_cron_enabled = False
 
     def is_extension_ok_for_version(self, extension, version):
         """Check if extension can be installed with this modo version."""
@@ -206,6 +207,10 @@ class Modoboa(base.Installer):
             packages += ["openssl-devel"]
         return packages
 
+    def setup_user(self):
+        super().setup_user()
+        self._setup_venv()
+
     def get_config_files(self):
         """Return appropriate path."""
         config_files = super().get_config_files()
@@ -214,6 +219,11 @@ class Modoboa(base.Installer):
         else:
             path = "supervisor=/etc/supervisord.d/policyd.ini"
         config_files.append(path)
+
+        # Add worker for dkim if needed
+        if self.modoboa_2_2_or_greater:
+            config_files.append(
+                "supervisor-rq=/etc/supervisor/conf.d/modoboa-worker.conf")
         return config_files
 
     def get_template_context(self):
@@ -222,6 +232,8 @@ class Modoboa(base.Installer):
         extensions = self.config.get("modoboa", "extensions")
         extensions = extensions.split()
         random_hour = random.randint(0, 6)
+        self.dkim_cron_enabled = (not self.modoboa_2_2_or_greater and
+                                  self.opendkim_enabled)
         context.update({
             "sudo_user": (
                 "uwsgi" if package.backend.FORMAT == "rpm" else context["user"]
@@ -232,7 +244,9 @@ class Modoboa(base.Installer):
                 "" if "modoboa-radicale" in extensions else "#"),
             "opendkim_user": self.config.get("opendkim", "user"),
             "minutes": random.randint(1, 59),
-            "hours" : f"{random_hour},{random_hour+12}"
+            "hours": f"{random_hour},{random_hour+12}",
+            "modoboa_2_2_or_greater": "" if self.modoboa_2_2_or_greater else "#",
+            "dkim_cron_enabled": "" if self.dkim_cron_enabled else "#"
         })
         return context
 
@@ -282,7 +296,6 @@ class Modoboa(base.Installer):
 
     def post_run(self):
         """Additional tasks."""
-        self._setup_venv()
         self._deploy_instance()
         if not self.upgrade:
             self.apply_settings()

modoboa_installer/utils.py

@@ -42,13 +42,15 @@ def user_input(message):
     return answer
 
 
-def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs):
+def exec_cmd(cmd, sudo_user=None, login=True, **kwargs):
-    """Execute a shell command.
+    """
+    Execute a shell command.
+
     Run a command using the current user. Set :keyword:`sudo_user` if
     you need different privileges.
+
     :param str cmd: the command to execute
     :param str sudo_user: a valid system username
-    :param str pinput: data to send to process's stdin
     :rtype: tuple
     :return: return code, command output
     """
@@ -57,23 +59,21 @@ def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs):
         cmd = "sudo {}-u {} {}".format("-i " if login else "", sudo_user, cmd)
     if "shell" not in kwargs:
         kwargs["shell"] = True
-    if pinput is not None:
+    capture_output = True
-        kwargs["stdin"] = subprocess.PIPE
-    capture_output = False
     if "capture_output" in kwargs:
         capture_output = kwargs.pop("capture_output")
-    elif not ENV.get("debug"):
-        capture_output = True
     if capture_output:
-        kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    output = None
+    kwargs["universal_newlines"] = True
-    process = subprocess.Popen(cmd, **kwargs)
+    output: str = ""
-    if pinput or capture_output:
+    with subprocess.Popen(cmd, **kwargs) as process:
-        c_args = [pinput] if pinput is not None else []
+        if capture_output:
-        output = process.communicate(*c_args)[0]
+            for line in process.stdout:
-    else:
+                output += line
-        process.wait()
+                if ENV.get("debug"):
-    return process.returncode, output
+                    sys.stdout.write(line)
+
+    return process.returncode, output.encode()
 
 
 def dist_info():
@@ -135,7 +135,6 @@ def settings(**kwargs):
 
 
 class ConfigFileTemplate(string.Template):
-
     """Custom class for configuration files."""
 
     delimiter = "%"