From 0b0e2a4e6aabb17ef52135c3ef5a1f13c0a57a65 Mon Sep 17 00:00:00 2001 From: Spitap Date: Tue, 25 Apr 2023 17:57:47 +0200 Subject: [PATCH 01/12] Updated for 2.2 --- modoboa_installer/python.py | 23 ++++++++++++++ modoboa_installer/scripts/dovecot.py | 16 +++++++++- .../dovecot/dovecot-sql-mysql.conf.ext.tpl | 6 ++-- .../dovecot/dovecot-sql-postgres.conf.ext.tpl | 6 ++-- .../scripts/files/modoboa/crontab.tpl | 2 +- .../scripts/files/modoboa/supervisor-rq.tpl | 9 ++++++ .../scripts/files/modoboa/supervisor.tpl | 1 + modoboa_installer/scripts/modoboa.py | 30 ++++++++++++++++++- 8 files changed, 86 insertions(+), 7 deletions(-) create mode 100644 modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl diff --git a/modoboa_installer/python.py b/modoboa_installer/python.py index 04788ca..f7f00c1 100644 --- a/modoboa_installer/python.py +++ b/modoboa_installer/python.py @@ -45,6 +45,29 @@ def install_packages(names, venv=None, upgrade=False, **kwargs): utils.exec_cmd(cmd, **kwargs) +def get_package_version(name, venv=None, **kwargs): + """Returns the version of an installed package.""" + cmd = "{} show {}".format( + get_pip_path(venv), + name + ) + status, output = utils.exec_cmd(cmd, **kwargs) + + output_list = output.split("\n") + version_item_list = output_list[1].split(":") + version_list = version_item_list[1].split(".") + version_list_clean = [] + for element in version_list: + try: + version_list_clean.append(int(version_list[i])) + except ValueError: + utils.printcolor( + f"Failed to decode some part of the version of {name}", + utils.YELLOW) + version_list_clean.append(element) + return version_list_clean + + def install_package_from_repository(name, url, vcs="git", venv=None, **kwargs): """Install a Python package from its repository.""" if vcs == "git": diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index ce4fa0f..3224de3 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -30,6 +30,17 @@ class Dovecot(base.Installer): "conf.d/10-master.conf", "conf.d/20-lmtp.conf", "conf.d/10-ssl-keys.try"] with_user = True + def __init__(self, *args, **kwargs): + super.__init__(*args, **kwargs) + # Check if modoboa version > 2.2 + self.modoboa_2_2_or_greater = False + condition = ( + (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or + modoboa_version[0] > 2 + ) + if condition: + self.modoboa_2_2_or_greater = True + def setup_user(self): """Setup mailbox user.""" super().setup_user() @@ -83,6 +94,7 @@ class Dovecot(base.Installer): else: # Protocols are automatically guessed on debian/ubuntu protocols = "" + context.update({ "db_driver": self.db_driver, "mailboxes_owner_uid": pw_mailbox[2], @@ -97,7 +109,9 @@ class Dovecot(base.Installer): "ssl_protocol_parameter": ssl_protocol_parameter, "radicale_user": self.config.get("radicale", "user"), "radicale_auth_socket_path": os.path.basename( - self.config.get("dovecot", "radicale_auth_socket_path")) + self.config.get("dovecot", "radicale_auth_socket_path")), + "modoboa_2_2_or_greater": "" if self.modoboa_2_2_or_greater else "#", + "not_modoboa_2_2_or_greater": "" if not self.modoboa_2_2_or_greater else "#" }) return context diff --git a/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl b/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl index 4538b9a..1a0434f 100644 --- a/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl +++ b/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl @@ -123,7 +123,8 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser #user_query = \ # SELECT home, uid, gid \ # FROM users WHERE username = '%%n' AND domain = '%%d' -user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' +%{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' +%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only=0 OR '%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d' # If you wish to avoid two SQL lookups (passdb + userdb), you can use # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll @@ -133,7 +134,8 @@ user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, # SELECT userid AS user, password, \ # home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \ # FROM users WHERE userid = '%%u' -password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE u.email='%%u' AND u.is_active=1 AND dom.enabled=1 +%{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE u.email='%%u' AND u.is_active=1 AND dom.enabled=1 +%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only=0 OR '%s' NOT IN ('imap', 'pop3')) AND u.email='%%u' AND u.is_active=1 AND dom.enabled=1 # Query to get a list of all usernames. #iterate_query = SELECT username AS user FROM users diff --git a/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl b/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl index abb5379..811e0f0 100644 --- a/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl +++ b/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl @@ -123,7 +123,8 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser #user_query = \ # SELECT home, uid, gid \ # FROM users WHERE username = '%%n' AND domain = '%%d' -user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' +%{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' +%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only IS NOT TRUE OR '%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d' # If you wish to avoid two SQL lookups (passdb + userdb), you can use # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll @@ -133,7 +134,8 @@ user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, # SELECT userid AS user, password, \ # home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \ # FROM users WHERE userid = '%%u' -password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE email='%%u' AND is_active AND dom.enabled +%{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE email='%%u' AND is_active AND dom.enabled +%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only IS NOT TRUE OR '%s' NOT IN ('imap', 'pop3')) AND email='%%u' AND is_active AND dom.enabled # Query to get a list of all usernames. #iterate_query = SELECT username AS user FROM users diff --git a/modoboa_installer/scripts/files/modoboa/crontab.tpl b/modoboa_installer/scripts/files/modoboa/crontab.tpl index 28bde36..84d9427 100644 --- a/modoboa_installer/scripts/files/modoboa/crontab.tpl +++ b/modoboa_installer/scripts/files/modoboa/crontab.tpl @@ -33,4 +33,4 @@ INSTANCE=%{instance_path} %{minutes} %{hours} * * * root $PYTHON $INSTANCE/manage.py communicate_with_public_api # Generate DKIM keys (they will belong to the user running this job) -%{opendkim_enabled}* * * * * %{opendkim_user} umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys +%{dkim_cron_enabled}* * * * * %{opendkim_user} umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys diff --git a/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl b/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl new file mode 100644 index 0000000..531650e --- /dev/null +++ b/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl @@ -0,0 +1,9 @@ +[program:modoboa-dkim-worker] +autostart=true +autorestart=true +command=%{venv_path}/bin/python %{home_dir}/instance/manage.py rqworker dkim +directory=%{home_dir} +user=%{dkim_user} +redirect_stderr=true +numprocs=1 +stopsignal=TERM diff --git a/modoboa_installer/scripts/files/modoboa/supervisor.tpl b/modoboa_installer/scripts/files/modoboa/supervisor.tpl index 0303a04..46e22fe 100644 --- a/modoboa_installer/scripts/files/modoboa/supervisor.tpl +++ b/modoboa_installer/scripts/files/modoboa/supervisor.tpl @@ -6,3 +6,4 @@ directory=%{home_dir} redirect_stderr=true user=%{user} numprocs=1 + diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 838d1b5..1496fb6 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -45,6 +45,15 @@ class Modoboa(base.Installer): def __init__(self, *args, **kwargs): """Get configuration.""" super(Modoboa, self).__init__(*args, **kwargs) + # Check if modoboa version > 2.2 + self.modoboa_2_2_or_greater = False + condition = ( + (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or + modoboa_version[0] > 2 + ) + if condition: + self.modoboa_2_2_or_greater = True + self.venv_path = self.config.get("modoboa", "venv_path") self.instance_path = self.config.get("modoboa", "instance_path") self.extensions = self.config.get("modoboa", "extensions").split() @@ -61,6 +70,9 @@ class Modoboa(base.Installer): self.extensions.remove("modoboa-radicale") self.dovecot_enabled = self.config.getboolean("dovecot", "enabled") self.opendkim_enabled = self.config.getboolean("opendkim", "enabled") + self.dkim_cron_enabled = False + if not self.modoboa_2_2_or_greater and self.opendkim_enabled: + self.dkim_cron_enabled = True def is_extension_ok_for_version(self, extension, version): """Check if extension can be installed with this modo version.""" @@ -214,6 +226,20 @@ class Modoboa(base.Installer): else: path = "supervisor=/etc/supervisord.d/policyd.ini" config_files.append(path) + + # Add worker for dkim if needed + modoboa_version = python.get_package_version( + "modoboa", + self.venv_path, + sudo_user=self.user + ) + condition = ( + (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or + modoboa_version[0] > 2 + ) + if condition: + config_files.append( + "supervisor-rq=/etc/supervisor/conf.d/modoboa-worker.conf") return config_files def get_template_context(self): @@ -232,7 +258,9 @@ class Modoboa(base.Installer): "" if "modoboa-radicale" in extensions else "#"), "opendkim_user": self.config.get("opendkim", "user"), "minutes": random.randint(1, 59), - "hours" : f"{random_hour},{random_hour+12}" + "hours": f"{random_hour},{random_hour+12}", + "modoboa_2_2_or_greater": "" if self.modoboa_2_2_or_greater else "#", + "dkim_cron_enabled": "" if self.dkim_cron_enabled else "#" }) return context From 35fa19e47d7a5262b57972d15f91349569f1ad49 Mon Sep 17 00:00:00 2001 From: Spitap Date: Tue, 29 Aug 2023 20:41:01 +0200 Subject: [PATCH 02/12] Cleaning code --- modoboa_installer/python.py | 7 ++++++- modoboa_installer/scripts/modoboa.py | 11 +---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/modoboa_installer/python.py b/modoboa_installer/python.py index f7f00c1..9b46ccf 100644 --- a/modoboa_installer/python.py +++ b/modoboa_installer/python.py @@ -1,6 +1,7 @@ """Python related tools.""" import os +import sys from . import package from . import utils @@ -51,7 +52,11 @@ def get_package_version(name, venv=None, **kwargs): get_pip_path(venv), name ) - status, output = utils.exec_cmd(cmd, **kwargs) + exit_code, output = utils.exec_cmd(cmd, **kwargs) + if exit_code != 0: + utils.error(f"Failed to get version of {name}. " + f"Output is: {output}") + sys.exit(1) output_list = output.split("\n") version_item_list = output_list[1].split(":") diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 1496fb6..7e8d01c 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -228,16 +228,7 @@ class Modoboa(base.Installer): config_files.append(path) # Add worker for dkim if needed - modoboa_version = python.get_package_version( - "modoboa", - self.venv_path, - sudo_user=self.user - ) - condition = ( - (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or - modoboa_version[0] > 2 - ) - if condition: + if self.modoboa_2_2_or_greater: config_files.append( "supervisor-rq=/etc/supervisor/conf.d/modoboa-worker.conf") return config_files From ef1bace29eb4a5ff6020deac3865c8b382e06834 Mon Sep 17 00:00:00 2001 From: Spitap Date: Tue, 29 Aug 2023 20:42:44 +0200 Subject: [PATCH 03/12] Cleaning code the 2nd --- modoboa_installer/scripts/dovecot.py | 6 ++++++ modoboa_installer/scripts/modoboa.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 3224de3..a5ba5c3 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -7,6 +7,7 @@ import shutil from .. import database from .. import package +from .. import python from .. import system from .. import utils @@ -33,6 +34,11 @@ class Dovecot(base.Installer): def __init__(self, *args, **kwargs): super.__init__(*args, **kwargs) # Check if modoboa version > 2.2 + modoboa_version = python.get_package_version( + "modoboa", + self.venv_path, + sudo_user=self.user + ) self.modoboa_2_2_or_greater = False condition = ( (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 7e8d01c..2414eb5 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -46,6 +46,11 @@ class Modoboa(base.Installer): """Get configuration.""" super(Modoboa, self).__init__(*args, **kwargs) # Check if modoboa version > 2.2 + modoboa_version = python.get_package_version( + "modoboa", + self.venv_path, + sudo_user=self.user + ) self.modoboa_2_2_or_greater = False condition = ( (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or From 941142f5f52c8a945344872da0cb57223daf3e5f Mon Sep 17 00:00:00 2001 From: Spitap Date: Wed, 30 Aug 2023 08:48:58 +0200 Subject: [PATCH 04/12] Fixed dkim user --- modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl b/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl index 531650e..42d9bd0 100644 --- a/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl +++ b/modoboa_installer/scripts/files/modoboa/supervisor-rq.tpl @@ -3,7 +3,7 @@ autostart=true autorestart=true command=%{venv_path}/bin/python %{home_dir}/instance/manage.py rqworker dkim directory=%{home_dir} -user=%{dkim_user} +user=%{opendkim_user} redirect_stderr=true numprocs=1 stopsignal=TERM From b1da76cfbdc83ec13de922374abcceb781badd40 Mon Sep 17 00:00:00 2001 From: Spitap Date: Wed, 30 Aug 2023 09:05:42 +0200 Subject: [PATCH 05/12] Fixed venvpath --- modoboa_installer/scripts/dovecot.py | 2 +- modoboa_installer/scripts/modoboa.py | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index a5ba5c3..8f0ca08 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -36,7 +36,7 @@ class Dovecot(base.Installer): # Check if modoboa version > 2.2 modoboa_version = python.get_package_version( "modoboa", - self.venv_path, + self.config.get("modoboa", "venv_path"), sudo_user=self.user ) self.modoboa_2_2_or_greater = False diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 2414eb5..2b36c8c 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -45,20 +45,6 @@ class Modoboa(base.Installer): def __init__(self, *args, **kwargs): """Get configuration.""" super(Modoboa, self).__init__(*args, **kwargs) - # Check if modoboa version > 2.2 - modoboa_version = python.get_package_version( - "modoboa", - self.venv_path, - sudo_user=self.user - ) - self.modoboa_2_2_or_greater = False - condition = ( - (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or - modoboa_version[0] > 2 - ) - if condition: - self.modoboa_2_2_or_greater = True - self.venv_path = self.config.get("modoboa", "venv_path") self.instance_path = self.config.get("modoboa", "instance_path") self.extensions = self.config.get("modoboa", "extensions").split() @@ -76,6 +62,20 @@ class Modoboa(base.Installer): self.dovecot_enabled = self.config.getboolean("dovecot", "enabled") self.opendkim_enabled = self.config.getboolean("opendkim", "enabled") self.dkim_cron_enabled = False + + # Check if modoboa version > 2.2 + modoboa_version = python.get_package_version( + "modoboa", + self.venv_path, + sudo_user=self.user + ) + self.modoboa_2_2_or_greater = False + condition = ( + (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or + modoboa_version[0] > 2 + ) + if condition: + self.modoboa_2_2_or_greater = True if not self.modoboa_2_2_or_greater and self.opendkim_enabled: self.dkim_cron_enabled = True From 1a528282ce474f7c18165d9d9d06f7dce0dc45e5 Mon Sep 17 00:00:00 2001 From: Spitap Date: Wed, 30 Aug 2023 09:47:10 +0200 Subject: [PATCH 06/12] Removed duplicates --- modoboa_installer/scripts/base.py | 14 ++++++++++++++ modoboa_installer/scripts/dovecot.py | 16 ---------------- modoboa_installer/scripts/modoboa.py | 18 ++---------------- 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 104ba4f..ef775e3 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -42,6 +42,20 @@ class Installer(object): self.dbuser = self.config.get(self.appname, "dbuser") self.dbpasswd = self.config.get(self.appname, "dbpassword") + @property + def modoboa_2_2_or_greater(self): + # Check if modoboa version > 2.2 + modoboa_version = python.get_package_version( + "modoboa", + self.self.config.get("modoboa", "venv_path"), + sudo_user=self.config.get("modoboa", "user") + ) + condition = ( + (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or + modoboa_version[0] > 2 + ) + return condition + @property def config_dir(self): """Return main configuration directory.""" diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 8f0ca08..321a342 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -31,22 +31,6 @@ class Dovecot(base.Installer): "conf.d/10-master.conf", "conf.d/20-lmtp.conf", "conf.d/10-ssl-keys.try"] with_user = True - def __init__(self, *args, **kwargs): - super.__init__(*args, **kwargs) - # Check if modoboa version > 2.2 - modoboa_version = python.get_package_version( - "modoboa", - self.config.get("modoboa", "venv_path"), - sudo_user=self.user - ) - self.modoboa_2_2_or_greater = False - condition = ( - (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or - modoboa_version[0] > 2 - ) - if condition: - self.modoboa_2_2_or_greater = True - def setup_user(self): """Setup mailbox user.""" super().setup_user() diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 2b36c8c..a2f683c 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -63,22 +63,6 @@ class Modoboa(base.Installer): self.opendkim_enabled = self.config.getboolean("opendkim", "enabled") self.dkim_cron_enabled = False - # Check if modoboa version > 2.2 - modoboa_version = python.get_package_version( - "modoboa", - self.venv_path, - sudo_user=self.user - ) - self.modoboa_2_2_or_greater = False - condition = ( - (modoboa_version[0] == 2 and modoboa_version[1] >= 2) or - modoboa_version[0] > 2 - ) - if condition: - self.modoboa_2_2_or_greater = True - if not self.modoboa_2_2_or_greater and self.opendkim_enabled: - self.dkim_cron_enabled = True - def is_extension_ok_for_version(self, extension, version): """Check if extension can be installed with this modo version.""" if extension not in compatibility_matrix.EXTENSIONS_AVAILABILITY: @@ -244,6 +228,8 @@ class Modoboa(base.Installer): extensions = self.config.get("modoboa", "extensions") extensions = extensions.split() random_hour = random.randint(0, 6) + self.dkim_cron_enabled = (not self.modoboa_2_2_or_greater and + self.opendkim_enabled) context.update({ "sudo_user": ( "uwsgi" if package.backend.FORMAT == "rpm" else context["user"] From 8a0b3cda9e9908e7fef841e24d51912ae5863345 Mon Sep 17 00:00:00 2001 From: Spitap Date: Wed, 30 Aug 2023 09:53:26 +0200 Subject: [PATCH 07/12] Added python module to base.py --- modoboa_installer/scripts/base.py | 1 + modoboa_installer/scripts/dovecot.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index ef775e3..f95987c 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -5,6 +5,7 @@ import sys from .. import database from .. import package +from .. import python from .. import system from .. import utils diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 321a342..b4520e1 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -7,7 +7,6 @@ import shutil from .. import database from .. import package -from .. import python from .. import system from .. import utils From 23a6101b7abd0e602f86af051655efdb893041cd Mon Sep 17 00:00:00 2001 From: Spitap Date: Wed, 30 Aug 2023 09:58:18 +0200 Subject: [PATCH 08/12] fix --- modoboa_installer/scripts/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index f95987c..c15cc91 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -48,7 +48,7 @@ class Installer(object): # Check if modoboa version > 2.2 modoboa_version = python.get_package_version( "modoboa", - self.self.config.get("modoboa", "venv_path"), + self.config.get("modoboa", "venv_path"), sudo_user=self.config.get("modoboa", "user") ) condition = ( From 4782000791b497de7f85b3b3e23849f9987f6323 Mon Sep 17 00:00:00 2001 From: Spitap Date: Wed, 30 Aug 2023 10:13:49 +0200 Subject: [PATCH 09/12] few fixes --- modoboa_installer/python.py | 2 +- modoboa_installer/scripts/modoboa.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modoboa_installer/python.py b/modoboa_installer/python.py index 9b46ccf..98390c5 100644 --- a/modoboa_installer/python.py +++ b/modoboa_installer/python.py @@ -58,7 +58,7 @@ def get_package_version(name, venv=None, **kwargs): f"Output is: {output}") sys.exit(1) - output_list = output.split("\n") + output_list = output.decode().split("\n") version_item_list = output_list[1].split(":") version_list = version_item_list[1].split(".") version_list_clean = [] diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index a2f683c..55d3ad1 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -207,6 +207,10 @@ class Modoboa(base.Installer): packages += ["openssl-devel"] return packages + def setup_user(self): + super().setup_user() + self._setup_venv() + def get_config_files(self): """Return appropriate path.""" config_files = super().get_config_files() @@ -292,7 +296,6 @@ class Modoboa(base.Installer): def post_run(self): """Additional tasks.""" - self._setup_venv() self._deploy_instance() if not self.upgrade: self.apply_settings() From 23aabbfffcbdab25e65c6c08daa41169d8df8a1e Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 30 Aug 2023 14:17:04 +0200 Subject: [PATCH 10/12] Updated exec_cmd to allow capturing while in debug mode --- modoboa_installer/package.py | 4 ++-- modoboa_installer/utils.py | 35 +++++++++++++++++------------------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/modoboa_installer/package.py b/modoboa_installer/package.py index e18d69d..2f2f3a2 100644 --- a/modoboa_installer/package.py +++ b/modoboa_installer/package.py @@ -68,7 +68,7 @@ class DEBPackage(Package): def get_installed_version(self, name): """Get installed package version.""" code, output = utils.exec_cmd( - "dpkg -s {} | grep Version".format(name), capture_output=True) + "dpkg -s {} | grep Version".format(name)) match = re.match(r"Version: (\d:)?(.+)-\d", output.decode()) if match: return match.group(2) @@ -97,7 +97,7 @@ class RPMPackage(Package): def get_installed_version(self, name): """Get installed package version.""" code, output = utils.exec_cmd( - "rpm -qi {} | grep Version".format(name), capture_output=True) + "rpm -qi {} | grep Version".format(name)) match = re.match(r"Version\s+: (.+)", output.decode()) if match: return match.group(1) diff --git a/modoboa_installer/utils.py b/modoboa_installer/utils.py index ded7e7d..2332273 100644 --- a/modoboa_installer/utils.py +++ b/modoboa_installer/utils.py @@ -42,13 +42,15 @@ def user_input(message): return answer -def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs): - """Execute a shell command. +def exec_cmd(cmd, sudo_user=None, login=True, **kwargs): + """ + Execute a shell command. + Run a command using the current user. Set :keyword:`sudo_user` if you need different privileges. + :param str cmd: the command to execute :param str sudo_user: a valid system username - :param str pinput: data to send to process's stdin :rtype: tuple :return: return code, command output """ @@ -57,23 +59,21 @@ def exec_cmd(cmd, sudo_user=None, pinput=None, login=True, **kwargs): cmd = "sudo {}-u {} {}".format("-i " if login else "", sudo_user, cmd) if "shell" not in kwargs: kwargs["shell"] = True - if pinput is not None: - kwargs["stdin"] = subprocess.PIPE - capture_output = False + capture_output = True if "capture_output" in kwargs: capture_output = kwargs.pop("capture_output") - elif not ENV.get("debug"): - capture_output = True if capture_output: - kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = None - process = subprocess.Popen(cmd, **kwargs) - if pinput or capture_output: - c_args = [pinput] if pinput is not None else [] - output = process.communicate(*c_args)[0] - else: - process.wait() - return process.returncode, output + kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + kwargs["universal_newlines"] = True + output: str = "" + with subprocess.Popen(cmd, **kwargs) as process: + if capture_output: + for line in process.stdout: + output += line + if ENV.get("debug"): + sys.stdout.write(line) + + return process.returncode, output.encode() def dist_info(): @@ -135,7 +135,6 @@ def settings(**kwargs): class ConfigFileTemplate(string.Template): - """Custom class for configuration files.""" delimiter = "%" From 9715fcc86e3d6c2b255e104a33a339d91c40b6b2 Mon Sep 17 00:00:00 2001 From: Spitap Date: Wed, 30 Aug 2023 16:57:49 +0200 Subject: [PATCH 11/12] few fixes --- modoboa_installer/package.py | 6 +++--- modoboa_installer/python.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modoboa_installer/package.py b/modoboa_installer/package.py index 2f2f3a2..5354b35 100644 --- a/modoboa_installer/package.py +++ b/modoboa_installer/package.py @@ -46,7 +46,7 @@ class DEBPackage(Package): """Update local cache.""" if self.index_updated: return - utils.exec_cmd("apt-get update --quiet") + utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 update --quiet") self.index_updated = True def preconfigure(self, name, question, qtype, answer): @@ -57,12 +57,12 @@ class DEBPackage(Package): def install(self, name): """Install a package.""" self.update() - utils.exec_cmd("apt-get install --quiet --assume-yes {}".format(name)) + utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 install --quiet --assume-yes {}".format(name)) def install_many(self, names): """Install many packages.""" self.update() - return utils.exec_cmd("apt-get install --quiet --assume-yes {}".format( + return utils.exec_cmd("apt-get -o Dpkg::Progress-Fancy=0 install --quiet --assume-yes {}".format( " ".join(names))) def get_installed_version(self, name): diff --git a/modoboa_installer/python.py b/modoboa_installer/python.py index 98390c5..4f319cc 100644 --- a/modoboa_installer/python.py +++ b/modoboa_installer/python.py @@ -64,7 +64,7 @@ def get_package_version(name, venv=None, **kwargs): version_list_clean = [] for element in version_list: try: - version_list_clean.append(int(version_list[i])) + version_list_clean.append(int(element)) except ValueError: utils.printcolor( f"Failed to decode some part of the version of {name}", From f658e5e85ee207397edf544fce733fdc7a980d10 Mon Sep 17 00:00:00 2001 From: Spitap Date: Wed, 30 Aug 2023 17:13:59 +0200 Subject: [PATCH 12/12] Fixed escape character on dovecot config tpl --- .../scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl | 4 ++-- .../scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl b/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl index 1a0434f..9cd3862 100644 --- a/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl +++ b/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl @@ -124,7 +124,7 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser # SELECT home, uid, gid \ # FROM users WHERE username = '%%n' AND domain = '%%d' %{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' -%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only=0 OR '%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d' +%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only=0 OR '%%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d' # If you wish to avoid two SQL lookups (passdb + userdb), you can use # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll @@ -135,7 +135,7 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser # home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \ # FROM users WHERE userid = '%%u' %{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE u.email='%%u' AND u.is_active=1 AND dom.enabled=1 -%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only=0 OR '%s' NOT IN ('imap', 'pop3')) AND u.email='%%u' AND u.is_active=1 AND dom.enabled=1 +%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only=0 OR '%%s' NOT IN ('imap', 'pop3')) AND u.email='%%u' AND u.is_active=1 AND dom.enabled=1 # Query to get a list of all usernames. #iterate_query = SELECT username AS user FROM users diff --git a/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl b/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl index 811e0f0..ac585fc 100644 --- a/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl +++ b/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl @@ -124,7 +124,7 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser # SELECT home, uid, gid \ # FROM users WHERE username = '%%n' AND domain = '%%d' %{not_modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' -%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only IS NOT TRUE OR '%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d' +%{modoboa_2_2_or_greater}user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE (mb.is_send_only IS NOT TRUE OR '%%s' NOT IN ('imap', 'pop3', 'lmtp')) AND mb.address='%%n' AND dom.name='%%d' # If you wish to avoid two SQL lookups (passdb + userdb), you can use # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll @@ -135,7 +135,7 @@ connect = host=%dbhost port=%dbport dbname=%modoboa_dbname user=%modoboa_dbuser # home AS userdb_home, uid AS userdb_uid, gid AS userdb_gid \ # FROM users WHERE userid = '%%u' %{not_modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE email='%%u' AND is_active AND dom.enabled -%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only IS NOT TRUE OR '%s' NOT IN ('imap', 'pop3')) AND email='%%u' AND is_active AND dom.enabled +%{modoboa_2_2_or_greater}password_query = SELECT email AS user, password, '%{home_dir}/%%d/%%n' AS userdb_home, %mailboxes_owner_uid AS userdb_uid, %mailboxes_owner_gid AS userdb_gid, CONCAT('*:bytes=', mb.quota, 'M') AS userdb_quota_rule FROM core_user u INNER JOIN admin_mailbox mb ON u.id=mb.user_id INNER JOIN admin_domain dom ON mb.domain_id=dom.id WHERE (mb.is_send_only IS NOT TRUE OR '%%s' NOT IN ('imap', 'pop3')) AND email='%%u' AND is_active AND dom.enabled # Query to get a list of all usernames. #iterate_query = SELECT username AS user FROM users