diff --git a/modoboa_installer/package.py b/modoboa_installer/package.py index e18d69d..5354b35 100644 --- a/modoboa_installer/package.py +++ b/modoboa_installer/package.py @@ -46,7 +46,7 @@ class DEBPackage(Package): """Update local cache.""" if self.index_updated: return - utils.exec_cmd("apt-get update --quiet") + utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 update --quiet") self.index_updated = True def preconfigure(self, name, question, qtype, answer): @@ -57,18 +57,18 @@ class DEBPackage(Package): def install(self, name): """Install a package.""" self.update() - utils.exec_cmd("apt-get install --quiet --assume-yes {}".format(name)) + utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 install --quiet --assume-yes {}".format(name)) def install_many(self, names): """Install many packages.""" self.update() - return utils.exec_cmd("apt-get install --quiet --assume-yes {}".format( + return utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 install --quiet --assume-yes {}".format( " ".join(names))) def get_installed_version(self, name): """Get installed package version.""" code, output = utils.exec_cmd( - "dpkg -s {} | grep Version".format(name), capture_output=True) + "dpkg -s {} | grep Version".format(name)) match = re.match(r"Version: (\d:)?(.+)-\d", output.decode()) if match: return match.group(2) @@ -97,7 +97,7 @@ class RPMPackage(Package): def get_installed_version(self, name): """Get installed package version.""" code, output = utils.exec_cmd( - "rpm -qi {} | grep Version".format(name), capture_output=True) + "rpm -qi {} | grep Version".format(name)) match = re.match(r"Version\s+: (.+)", output.decode()) if match: return match.group(1) diff --git a/modoboa_installer/python.py b/modoboa_installer/python.py index 04788ca..4f319cc 100644 --- a/modoboa_installer/python.py +++ b/modoboa_installer/python.py @@ -1,6 +1,7 @@ """Python related tools.""" import os +import sys from . import package from . import utils @@ -45,6 +46,33 @@ def install_packages(names, venv=None, upgrade=False, **kwargs): utils.exec_cmd(cmd, **kwargs) +def get_package_version(name, venv=None, **kwargs): + """Returns the version of an installed package.""" + cmd = "{} show {}".format( + get_pip_path(venv), + name + ) + exit_code, output = utils.exec_cmd(cmd, **kwargs) + if exit_code != 0: + utils.error(f"Failed to get version of {name}. " + f"Output is: {output}") + sys.exit(1) + + output_list = output.decode().split("\n") + version_item_list = output_list[1].split(":") + version_list = version_item_list[1].split(".") + version_list_clean = [] + for element in version_list: + try: + version_list_clean.append(int(element)) + except ValueError: + utils.printcolor( + f"Failed to decode some part of the version of {name}", + utils.YELLOW) + version_list_clean.append(element) + return version_list_clean + + def install_package_from_repository(name, url, vcs="git", venv=None, **kwargs): """Install a Python package from its repository.""" if vcs == "git": diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 104ba4f..c15cc91 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -5,6 +5,7 @@ import sys from .. import database from .. import package +from .. import python from .. import system from .. import utils @@ -42,6 +43,20 @@ class Installer(object): self.dbuser = self.config.get(self.appname, "dbuser") self.dbpasswd = self.config.get(self.appname, "dbpassword") + @property + def modoboa_2_2_or_greater(self): + # Check if modoboa version > 2.2 + modoboa_version = python.get_package_version( + "modoboa", + self.config.get("modoboa", "venv_path"), + sudo_user=self.config.get("modoboa", "user") + ) + condition = ( + (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or + modoboa_version[0] > 2 + ) + return condition + @property def config_dir(self): """Return main configuration directory.""" diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index ce4fa0f..b4520e1 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -83,6 +83,7 @@ class Dovecot(base.Installer): else: # Protocols are automatically guessed on debian/ubuntu protocols = "" + context.update({ "db_driver": self.db_driver, "mailboxes_owner_uid": pw_mailbox[2], @@ -97,7 +98,9 @@ class Dovecot(base.Installer): "ssl_protocol_parameter": ssl_protocol_parameter, "radicale_user": self.config.get("radicale", "user"), "radicale_auth_socket_path": os.path.basename( - self.config.get("dovecot", "radicale_auth_socket_path")) + self.config.get("dovecot", "radicale_auth_socket_path")), + "modoboa_2_2_or_greater": "" if self.modoboa_2_2_or_greater else "#", + "not_modoboa_2_2_or_greater": "" if not self.modoboa_2_2_or_greater else "#" }) return context diff --git a/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl b/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl index 4538b9a..9cd3862 100644 --- a/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl +++ b/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl @@ -123,7 +123,8 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser #user_query = \ # SELECT home, uid, gid \ # FROM users WHERE username = '%%n' AND domain = '%%d' -user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' +%{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' +%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only=0 OR '%%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d' # If you wish to avoid two SQL lookups (passdb + userdb), you can use # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll @@ -133,7 +134,8 @@ user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, # SELECT userid AS user, password, \ # home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \ # FROM users WHERE userid = '%%u' -password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE u.email='%%u' AND u.is_active=1 AND dom.enabled=1 +%{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE u.email='%%u' AND u.is_active=1 AND dom.enabled=1 +%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only=0 OR '%%s' NOT IN ('imap', 'pop3')) AND u.email='%%u' AND u.is_active=1 AND dom.enabled=1 # Query to get a list of all usernames. #iterate_query = SELECT username AS user FROM users diff --git a/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl b/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl index abb5379..ac585fc 100644 --- a/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl +++ b/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl @@ -123,7 +123,8 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser #user_query = \ # SELECT home, uid, gid \ # FROM users WHERE username = '%%n' AND domain = '%%d' -user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' +%{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' +%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only IS NOT TRUE OR '%%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d' # If you wish to avoid two SQL lookups (passdb + userdb), you can use # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll @@ -133,7 +134,8 @@ user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, # SELECT userid AS user, password, \ # home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \ # FROM users WHERE userid = '%%u' -password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE email='%%u' AND is_active AND dom.enabled +%{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE email='%%u' AND is_active AND dom.enabled +%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only IS NOT TRUE OR '%%s' NOT IN ('imap', 'pop3')) AND email='%%u' AND is_active AND dom.enabled # Query to get a list of all usernames. #iterate_query = SELECT username AS user FROM users diff --git a/modoboa_installer/scripts/files/modoboa/crontab.tpl b/modoboa_installer/scripts/files/modoboa/crontab.tpl index 28bde36..84d9427 100644 --- a/modoboa_installer/scripts/files/modoboa/crontab.tpl +++ b/modoboa_installer/scripts/files/modoboa/crontab.tpl @@ -33,4 +33,4 @@ INSTANCE=%{instance_path} %{minutes} %{hours} * * * root $PYTHON $INSTANCE/manage.py communicate_with_public_api # Generate DKIM keys (they will belong to the user running this job) -%{opendkim_enabled}* * * * * %{opendkim_user} umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys +%{dkim_cron_enabled}* * * * * %{opendkim_user} umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys diff --git a/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl b/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl new file mode 100644 index 0000000..42d9bd0 --- /dev/null +++ b/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl @@ -0,0 +1,9 @@ +[program:modoboa-dkim-worker] +autostart=true +autorestart=true +command=%{venv_path}/bin/python %{home_dir}/instance/manage.py rqworker dkim +directory=%{home_dir} +user=%{opendkim_user} +redirect_stderr=true +numprocs=1 +stopsignal=TERM diff --git a/modoboa_installer/scripts/files/modoboa/supervisor.tpl b/modoboa_installer/scripts/files/modoboa/supervisor.tpl index 0303a04..46e22fe 100644 --- a/modoboa_installer/scripts/files/modoboa/supervisor.tpl +++ b/modoboa_installer/scripts/files/modoboa/supervisor.tpl @@ -6,3 +6,4 @@ directory=%{home_dir} redirect_stderr=true user=%{user} numprocs=1 + diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 838d1b5..55d3ad1 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -61,6 +61,7 @@ class Modoboa(base.Installer): self.extensions.remove("modoboa-radicale") self.dovecot_enabled = self.config.getboolean("dovecot", "enabled") self.opendkim_enabled = self.config.getboolean("opendkim", "enabled") + self.dkim_cron_enabled = False def is_extension_ok_for_version(self, extension, version): """Check if extension can be installed with this modo version.""" @@ -206,6 +207,10 @@ class Modoboa(base.Installer): packages += ["openssl-devel"] return packages + def setup_user(self): + super().setup_user() + self._setup_venv() + def get_config_files(self): """Return appropriate path.""" config_files = super().get_config_files() @@ -214,6 +219,11 @@ class Modoboa(base.Installer): else: path = "supervisor=/etc/supervisord.d/policyd.ini" config_files.append(path) + + # Add worker for dkim if needed + if self.modoboa_2_2_or_greater: + config_files.append( + "supervisor-rq=/etc/supervisor/conf.d/modoboa-worker.conf") return config_files def get_template_context(self): @@ -222,6 +232,8 @@ class Modoboa(base.Installer): extensions = self.config.get("modoboa", "extensions") extensions = extensions.split() random_hour = random.randint(0, 6) + self.dkim_cron_enabled = (not self.modoboa_2_2_or_greater and + self.opendkim_enabled) context.update({ "sudo_user": ( "uwsgi" if package.backend.FORMAT == "rpm" else context["user"] @@ -232,7 +244,9 @@ class Modoboa(base.Installer): "" if "modoboa-radicale" in extensions else "#"), "opendkim_user": self.config.get("opendkim", "user"), "minutes": random.randint(1, 59), - "hours" : f"{random_hour},{random_hour+12}" + "hours": f"{random_hour},{random_hour+12}", + "modoboa_2_2_or_greater": "" if self.modoboa_2_2_or_greater else "#", + "dkim_cron_enabled": "" if self.dkim_cron_enabled else "#" }) return context @@ -282,7 +296,6 @@ class Modoboa(base.Installer): def post_run(self): """Additional tasks.""" - self._setup_venv() self._deploy_instance() if not self.upgrade: self.apply_settings() diff --git a/modoboa_installer/utils.py b/modoboa_installer/utils.py index ded7e7d..2332273 100644 --- a/modoboa_installer/utils.py +++ b/modoboa_installer/utils.py @@ -42,13 +42,15 @@ def user_input(message): return answer -def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs): - """Execute a shell command. +def exec_cmd(cmd, sudo_user=None, login=True, **kwargs): + """ + Execute a shell command. + Run a command using the current user. Set :keyword:`sudo_user` if you need different privileges. + :param str cmd: the command to execute :param str sudo_user: a valid system username - :param str pinput: data to send to process's stdin :rtype: tuple :return: return code, command output """ @@ -57,23 +59,21 @@ def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs): cmd = "sudo {}-u {} {}".format("-i " if login else "", sudo_user, cmd) if "shell" not in kwargs: kwargs["shell"] = True - if pinput is not None: - kwargs["stdin"] = subprocess.PIPE - capture_output = False + capture_output = True if "capture_output" in kwargs: capture_output = kwargs.pop("capture_output") - elif not ENV.get("debug"): - capture_output = True if capture_output: - kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = None - process = subprocess.Popen(cmd, **kwargs) - if pinput or capture_output: - c_args = [pinput] if pinput is not None else [] - output = process.communicate(*c_args)[0] - else: - process.wait() - return process.returncode, output + kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + kwargs["universal_newlines"] = True + output: str = "" + with subprocess.Popen(cmd, **kwargs) as process: + if capture_output: + for line in process.stdout: + output += line + if ENV.get("debug"): + sys.stdout.write(line) + + return process.returncode, output.encode() def dist_info(): @@ -135,7 +135,6 @@ def settings(**kwargs): class ConfigFileTemplate(string.Template): - """Custom class for configuration files.""" delimiter = "%"