From da7d45639fdf6cfc38347d0550dcd965bd66df0a Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 7 Jul 2017 10:36:38 +0200 Subject: [PATCH 01/56] Added support for modoboa version selection. see #138 --- modoboa_installer/compatibility_matrix.py | 8 ++++++ modoboa_installer/scripts/modoboa.py | 30 ++++++++++++++++++++++- modoboa_installer/utils.py | 29 ++++++++++++++++++++++ run.py | 11 +++++++-- 4 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 modoboa_installer/compatibility_matrix.py diff --git a/modoboa_installer/compatibility_matrix.py b/modoboa_installer/compatibility_matrix.py new file mode 100644 index 0000000..8da21a4 --- /dev/null +++ b/modoboa_installer/compatibility_matrix.py @@ -0,0 +1,8 @@ +"""Modoboa compatibility matrix.""" + +COMPATIBILITY_MATRIX = { +} + +EXTENSIONS_AVAILABILITY = { + "modoboa-contacts": "1.7.4", +} diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index a7b7512..6c8361e 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -7,6 +7,7 @@ import shutil import stat import sys +from .. import compatibility_matrix from .. import package from .. import python from .. import utils @@ -50,10 +51,36 @@ class Modoboa(base.Installer): else: self.extensions.remove("modoboa-amavis") + def is_extension_ok_for_version(self, extension, version): + """Check if extension can be installed with this modo version.""" + if extension not in compatibility_matrix.EXTENSIONS_AVAILABILITY: + return True + version = utils.convert_version_to_int(version) + min_version = compatibility_matrix.EXTENSIONS_AVAILABILITY[extension] + min_version = utils.convert_version_to_int(min_version) + return version >= min_version + def _setup_venv(self): """Prepare a dedicated virtualenv.""" python.setup_virtualenv(self.venv_path, sudo_user=self.user) - packages = ["modoboa", "rrdtool"] + packages = ["rrdtool"] + version = self.config.get("modoboa", "version") + if version == "latest": + packages += ["modoboa"] + self.extensions + else: + matrix = compatibility_matrix.COMPATIBILITY_MATRIX[version] + packages.append("modoboa=={}".format(version)) + for extension in list(self.extensions): + if not self.is_extension_ok_for_version(extension, version): + self.extensions.remove(extension) + continue + if extension in matrix: + req_version = matrix[extension] + req_version = req_version.replace("<", "\<") + req_version = req_version.replace(">", "\>") + packages.append("{}{}".format(extension, req_version)) + else: + packages.append(extension) if self.dbengine == "postgres": packages.append("psycopg2") else: @@ -91,6 +118,7 @@ class Modoboa(base.Installer): "--timezone", self.config.get("modoboa", "timezone"), "--domain", self.config.get("general", "hostname"), "--extensions", " ".join(self.extensions), + "--dont-install-extensions", "--dburl", "'default:{0}://{1}:{2}@{3}/{1}'".format( self.config.get("database", "engine"), self.dbname, self.dbpasswd, self.dbhost) diff --git a/modoboa_installer/utils.py b/modoboa_installer/utils.py index 68e43f4..e4baee8 100644 --- a/modoboa_installer/utils.py +++ b/modoboa_installer/utils.py @@ -171,6 +171,8 @@ def has_colours(stream): except: # guess false in case of error return False + + has_colours = has_colours(sys.stdout) @@ -179,3 +181,30 @@ def printcolor(message, color): if has_colours: message = "\x1b[1;{}m{}\x1b[0m".format(30 + color, message) print(message) + + +def convert_version_to_int(version): + """Convert a version string to an integer.""" + number_bits = (8, 8, 16) + + numbers = [int(number_string) for number_string in version.split(".")] + if len(numbers) > len(number_bits): + raise NotImplementedError( + "Versions with more than {0} decimal places are not supported" + .format(len(number_bits) - 1) + ) + # add 0s for missing numbers + numbers.extend([0] * (len(number_bits) - len(numbers))) + # convert to single int and return + number = 0 + total_bits = 0 + for num, bits in reversed(list(zip(numbers, number_bits))): + max_num = (bits + 1) - 1 + if num >= 1 << max_num: + raise ValueError( + "Number {0} cannot be stored with only {1} bits. Max is {2}" + .format(num, bits, max_num) + ) + number += num << total_bits + total_bits += bits + return number diff --git a/run.py b/run.py index e0f600b..385829e 100755 --- a/run.py +++ b/run.py @@ -8,10 +8,11 @@ try: except ImportError: import ConfigParser as configparser -from modoboa_installer import scripts -from modoboa_installer import utils +from modoboa_installer import compatibility_matrix from modoboa_installer import package +from modoboa_installer import scripts from modoboa_installer import ssl +from modoboa_installer import utils def main(): @@ -23,6 +24,10 @@ def main(): help="Force installation") parser.add_argument("--configfile", default="installer.cfg", help="Configuration file to use") + parser.add_argument( + "--version", default="latest", + choices=["latest"] + compatibility_matrix.COMPATIBILITY_MATRIX.keys(), + help="Modoboa version to install") parser.add_argument( "--stop-after-configfile-check", action="store_true", default=False, help="Check configuration, generate it if needed and exit") @@ -42,6 +47,7 @@ def main(): if not config.has_section("general"): config.add_section("general") config.set("general", "domain", args.domain) + config.set("modoboa", "version", args.version) utils.printcolor( "Your mail server will be installed with the following components:", utils.BLUE) @@ -80,5 +86,6 @@ def main(): .format(config.get("general", "hostname")), utils.GREEN) + if __name__ == "__main__": main() From a1d15edb05392ccb21d5fb3ac369fccf48cb745f Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 7 Jul 2017 18:27:55 +0200 Subject: [PATCH 02/56] Added example to compatibility matrix. see #138 --- modoboa_installer/compatibility_matrix.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modoboa_installer/compatibility_matrix.py b/modoboa_installer/compatibility_matrix.py index 8da21a4..b986e96 100644 --- a/modoboa_installer/compatibility_matrix.py +++ b/modoboa_installer/compatibility_matrix.py @@ -1,6 +1,9 @@ """Modoboa compatibility matrix.""" COMPATIBILITY_MATRIX = { + # Example: + # + # "1.8.1": {} } EXTENSIONS_AVAILABILITY = { From 8c46bbd9494b9d78eb235099b6464747dab14706 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 7 Jul 2017 18:34:06 +0200 Subject: [PATCH 03/56] Added documentation. see #138 --- README.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 978e460..91df716 100644 --- a/README.rst +++ b/README.rst @@ -31,13 +31,14 @@ A configuration file will be automatically generated the first time you run the installer, please don't copy the ``installer.cfg.template`` file manually. -By default, the following components are installed: +The following components are installed by the installer: * Database server (PostgreSQL or MySQL) * Nginx and uWSGI * Postfix * Dovecot * Amavis (with SpamAssassin and ClamAV) +* automx (autoconfiguration service) If you want to customize configuration before running the installer, run the following command:: @@ -46,6 +47,15 @@ run the following command:: Make your modifications and run the installer as usual. +By default, the latest Modoboa version is installed but you can select +a previous one using the ``--version`` option:: + + $ sudo ./run.py --version=X.X.X + +.. note:: + + Version selection is available only for Modoboa >= 1.8.1. + If you want more information about the installation process, add the ``--debug`` option to your command line. From af2bd9bde61faa30dc269d0322c10f825a870593 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 28 Jul 2017 16:26:40 +0200 Subject: [PATCH 04/56] Added missing db driver to automx env. fix #147 --- modoboa_installer/scripts/automx.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modoboa_installer/scripts/automx.py b/modoboa_installer/scripts/automx.py index e278645..89e67db 100644 --- a/modoboa_installer/scripts/automx.py +++ b/modoboa_installer/scripts/automx.py @@ -58,6 +58,10 @@ class Automx(base.Installer): "future", "lxml", "ipaddress", "sqlalchemy", "python-memcached", "python-dateutil", "configparser" ] + if self.dbengine == "postgres": + packages.append("psycopg2") + else: + packages.append("MYSQL-Python") python.install_packages(packages, self.venv_path, sudo_user=self.user) target = "{}/master.zip".format(self.home_dir) if os.path.exists(target): From cd3955839bd755a341c53b3262ac96602f87f5c1 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 28 Jul 2017 16:52:39 +0200 Subject: [PATCH 05/56] Enabled SA update cron job on Debian. fix #143 --- modoboa_installer/scripts/spamassassin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modoboa_installer/scripts/spamassassin.py b/modoboa_installer/scripts/spamassassin.py index 941961d..7b14f5f 100644 --- a/modoboa_installer/scripts/spamassassin.py +++ b/modoboa_installer/scripts/spamassassin.py @@ -62,3 +62,6 @@ class Spamassassin(base.Installer): sudo_user=amavis_user, login=False ) install("razor", self.config) + if utils.dist_name() in ["debian", "ubuntu"]: + utils.exec_cmd( + "perl -pi -e 's/^CRON=0/CRON=1/' /etc/cron.daily/spamassassin") From df194d1db3731b8be245a8adaef801e5491888f2 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 28 Jul 2017 16:54:27 +0200 Subject: [PATCH 06/56] Added amavis 2.11.0 SQL schemas. --- .../files/amavis/amavis_mysql_2.11.0.sql | 211 ++++++++++++++++++ .../files/amavis/amavis_postgres_2.11.0.sql | 187 ++++++++++++++++ 2 files changed, 398 insertions(+) create mode 100644 modoboa_installer/scripts/files/amavis/amavis_mysql_2.11.0.sql create mode 100644 modoboa_installer/scripts/files/amavis/amavis_postgres_2.11.0.sql diff --git a/modoboa_installer/scripts/files/amavis/amavis_mysql_2.11.0.sql b/modoboa_installer/scripts/files/amavis/amavis_mysql_2.11.0.sql new file mode 100644 index 0000000..8c9289c --- /dev/null +++ b/modoboa_installer/scripts/files/amavis/amavis_mysql_2.11.0.sql @@ -0,0 +1,211 @@ +-- Amavis 2.11.0 MySQL schema +-- Provided by Modoboa +-- Warning: foreign key creations are enabled + +-- local users +CREATE TABLE users ( + id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, -- unique id + priority integer NOT NULL DEFAULT '7', -- sort field, 0 is low prior. + policy_id integer unsigned NOT NULL DEFAULT '1', -- JOINs with policy.id + email varbinary(255) NOT NULL UNIQUE, + fullname varchar(255) DEFAULT NULL -- not used by amavisd-new + -- local char(1) -- Y/N (optional field, see note further down) +); + +-- any e-mail address (non- rfc2822-quoted), external or local, +-- used as senders in wblist +CREATE TABLE mailaddr ( + id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + priority integer NOT NULL DEFAULT '7', -- 0 is low priority + email varbinary(255) NOT NULL UNIQUE +); + +-- per-recipient whitelist and/or blacklist, +-- puts sender and recipient in relation wb (white or blacklisted sender) +CREATE TABLE wblist ( + rid integer unsigned NOT NULL, -- recipient: users.id + sid integer unsigned NOT NULL, -- sender: mailaddr.id + wb varchar(10) NOT NULL, -- W or Y / B or N / space=neutral / score + PRIMARY KEY (rid,sid) +); + +CREATE TABLE policy ( + id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + -- 'id' this is the _only_ required field + policy_name varchar(32), -- not used by amavisd-new, a comment + + virus_lover char(1) default NULL, -- Y/N + spam_lover char(1) default NULL, -- Y/N + unchecked_lover char(1) default NULL, -- Y/N + banned_files_lover char(1) default NULL, -- Y/N + bad_header_lover char(1) default NULL, -- Y/N + + bypass_virus_checks char(1) default NULL, -- Y/N + bypass_spam_checks char(1) default NULL, -- Y/N + bypass_banned_checks char(1) default NULL, -- Y/N + bypass_header_checks char(1) default NULL, -- Y/N + + virus_quarantine_to varchar(64) default NULL, + spam_quarantine_to varchar(64) default NULL, + banned_quarantine_to varchar(64) default NULL, + unchecked_quarantine_to varchar(64) default NULL, + bad_header_quarantine_to varchar(64) default NULL, + clean_quarantine_to varchar(64) default NULL, + archive_quarantine_to varchar(64) default NULL, + + spam_tag_level float default NULL, -- higher score inserts spam info headers + spam_tag2_level float default NULL, -- inserts 'declared spam' header fields + spam_tag3_level float default NULL, -- inserts 'blatant spam' header fields + spam_kill_level float default NULL, -- higher score triggers evasive actions + -- e.g. reject/drop, quarantine, ... + -- (subject to final_spam_destiny setting) + spam_dsn_cutoff_level float default NULL, + spam_quarantine_cutoff_level float default NULL, + + addr_extension_virus varchar(64) default NULL, + addr_extension_spam varchar(64) default NULL, + addr_extension_banned varchar(64) default NULL, + addr_extension_bad_header varchar(64) default NULL, + + warnvirusrecip char(1) default NULL, -- Y/N + warnbannedrecip char(1) default NULL, -- Y/N + warnbadhrecip char(1) default NULL, -- Y/N + newvirus_admin varchar(64) default NULL, + virus_admin varchar(64) default NULL, + banned_admin varchar(64) default NULL, + bad_header_admin varchar(64) default NULL, + spam_admin varchar(64) default NULL, + spam_subject_tag varchar(64) default NULL, + spam_subject_tag2 varchar(64) default NULL, + spam_subject_tag3 varchar(64) default NULL, + message_size_limit integer default NULL, -- max size in bytes, 0 disable + banned_rulenames varchar(64) default NULL, -- comma-separated list of ... + -- names mapped through %banned_rules to actual banned_filename tables + disclaimer_options varchar(64) default NULL, + forward_method varchar(64) default NULL, + sa_userconf varchar(64) default NULL, + sa_username varchar(64) default NULL +); + + +-- R/W part of the dataset (optional) +-- May reside in the same or in a separate database as lookups database; +-- REQUIRES SUPPORT FOR TRANSACTIONS; specified in @storage_sql_dsn +-- +-- MySQL note ( http://dev.mysql.com/doc/mysql/en/storage-engines.html ): +-- ENGINE is the preferred term, but cannot be used before MySQL 4.0.18. +-- TYPE is available beginning with MySQL 3.23.0, the first version of +-- MySQL for which multiple storage engines were available. If you omit +-- the ENGINE or TYPE option, the default storage engine is used. +-- By default this is MyISAM. +-- +-- Please create additional indexes on keys when needed, or drop suggested +-- ones as appropriate to optimize queries needed by a management application. +-- See your database documentation for further optimization hints. With MySQL +-- see Chapter 15 of the reference manual. For example the chapter 15.17 says: +-- InnoDB does not keep an internal count of rows in a table. To process a +-- SELECT COUNT(*) FROM T statement, InnoDB must scan an index of the table, +-- which takes some time if the index is not entirely in the buffer pool. +-- +-- Wayne Smith adds: When using MySQL with InnoDB one might want to +-- increase buffer size for both pool and log, and might also want +-- to change flush settings for a little better performance. Example: +-- innodb_buffer_pool_size = 384M +-- innodb_log_buffer_size = 8M +-- innodb_flush_log_at_trx_commit = 0 +-- The big performance increase is the first two, the third just helps with +-- lowering disk activity. Consider also adjusting the key_buffer_size. + +-- provide unique id for each e-mail address, avoids storing copies +CREATE TABLE maddr ( + partition_tag integer DEFAULT 0, -- see $partition_tag + id bigint unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + email varbinary(255) NOT NULL, -- full mail address + domain varchar(255) NOT NULL, -- only domain part of the email address + -- with subdomain fields in reverse + CONSTRAINT part_email UNIQUE (partition_tag,email) +) ENGINE=InnoDB; + +-- information pertaining to each processed message as a whole; +-- NOTE: records with NULL msgs.content should be ignored by utilities, +-- as such records correspond to messages just being processes, or were lost +-- NOTE: instead of a character field time_iso, one might prefer: +-- time_iso TIMESTAMP NOT NULL DEFAULT 0, +-- but the following MUST then be set in amavisd.conf: $timestamp_fmt_mysql=1 +CREATE TABLE msgs ( + partition_tag integer DEFAULT 0, -- see $partition_tag + mail_id varbinary(16) NOT NULL, -- long-term unique mail id, dflt 12 ch + secret_id varbinary(16) DEFAULT '', -- authorizes release of mail_id, 12 ch + am_id varchar(20) NOT NULL, -- id used in the log + time_num integer unsigned NOT NULL, -- rx_time: seconds since Unix epoch + time_iso char(16) NOT NULL, -- rx_time: ISO8601 UTC ascii time + sid bigint unsigned NOT NULL, -- sender: maddr.id + policy varchar(255) DEFAULT '', -- policy bank path (like macro %p) + client_addr varchar(255) DEFAULT '', -- SMTP client IP address (IPv4 or v6) + size integer unsigned NOT NULL, -- message size in bytes + originating char(1) DEFAULT ' ' NOT NULL, -- sender from inside or auth'd + content char(1), -- content type: V/B/U/S/Y/M/H/O/T/C + -- virus/banned/unchecked/spam(kill)/spammy(tag2)/ + -- /bad-mime/bad-header/oversized/mta-err/clean + -- is NULL on partially processed mail + -- (prior to 2.7.0 the CC_SPAMMY was logged as 's', now 'Y' is used; + -- to avoid a need for case-insenstivity in queries) + quar_type char(1), -- quarantined as: ' '/F/Z/B/Q/M/L + -- none/file/zipfile/bsmtp/sql/ + -- /mailbox(smtp)/mailbox(lmtp) + quar_loc varbinary(255) DEFAULT '', -- quarantine location (e.g. file) + dsn_sent char(1), -- was DSN sent? Y/N/q (q=quenched) + spam_level float, -- SA spam level (no boosts) + message_id varchar(255) DEFAULT '', -- mail Message-ID header field + from_addr varchar(255) CHARACTER SET utf8mb4 COLLATE utf8_bin DEFAULT '', + -- mail From header field, UTF8 + subject varchar(255) CHARACTER SET utf8mb4 COLLATE utf8_bin DEFAULT '', + -- mail Subject header field, UTF8 + host varchar(255) NOT NULL, -- hostname where amavisd is running + PRIMARY KEY (partition_tag,mail_id) + FOREIGN KEY (sid) REFERENCES maddr(id) ON DELETE RESTRICT +) ENGINE=InnoDB; +CREATE INDEX msgs_idx_sid ON msgs (sid); +CREATE INDEX msgs_idx_mess_id ON msgs (message_id); -- useful with pen pals +CREATE INDEX msgs_idx_time_num ON msgs (time_num); +-- alternatively when purging based on time_iso (instead of msgs_idx_time_num): +-- CREATE INDEX msgs_idx_time_iso ON msgs (time_iso); +-- When using FOREIGN KEY contraints, InnoDB requires index on a field +-- (an the field must be the first field in the index). Hence create it: +-- CREATE INDEX msgs_idx_mail_id ON msgs (mail_id); + +-- per-recipient information related to each processed message; +-- NOTE: records in msgrcpt without corresponding msgs.mail_id record are +-- orphaned and should be ignored and eventually deleted by external utilities +CREATE TABLE msgrcpt ( + partition_tag integer DEFAULT 0, -- see $partition_tag + mail_id varbinary(16) NOT NULL, -- (must allow duplicates) + rseqnum integer DEFAULT 0 NOT NULL, -- recip's enumeration within msg + rid bigint unsigned NOT NULL, -- recipient: maddr.id (dupl. allowed) + is_local char(1) DEFAULT ' ' NOT NULL, -- recip is: Y=local, N=foreign + content char(1) DEFAULT ' ' NOT NULL, -- content type V/B/U/S/Y/M/H/O/T/C + ds char(1) NOT NULL, -- delivery status: P/R/B/D/T + -- pass/reject/bounce/discard/tempfail + rs char(1) NOT NULL, -- release status: initialized to ' ' + bl char(1) DEFAULT ' ', -- sender blacklisted by this recip + wl char(1) DEFAULT ' ', -- sender whitelisted by this recip + bspam_level float, -- per-recipient (total) spam level + smtp_resp varchar(255) DEFAULT '', -- SMTP response given to MTA + PRIMARY KEY (partition_tag,mail_id,rseqnum) + FOREIGN KEY (rid) REFERENCES maddr(id) ON DELETE RESTRICT, + FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE +) ENGINE=InnoDB; +CREATE INDEX msgrcpt_idx_mail_id ON msgrcpt (mail_id); +CREATE INDEX msgrcpt_idx_rid ON msgrcpt (rid); + +-- mail quarantine in SQL, enabled by $*_quarantine_method='sql:' +-- NOTE: records in quarantine without corresponding msgs.mail_id record are +-- orphaned and should be ignored and eventually deleted by external utilities +CREATE TABLE quarantine ( + partition_tag integer DEFAULT 0, -- see $partition_tag + mail_id varbinary(16) NOT NULL, -- long-term unique mail id + chunk_ind integer unsigned NOT NULL, -- chunk number, starting with 1 + mail_text blob NOT NULL, -- store mail as chunks of octets + PRIMARY KEY (partition_tag,mail_id,chunk_ind) + FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE +) ENGINE=InnoDB; diff --git a/modoboa_installer/scripts/files/amavis/amavis_postgres_2.11.0.sql b/modoboa_installer/scripts/files/amavis/amavis_postgres_2.11.0.sql new file mode 100644 index 0000000..06cb3b0 --- /dev/null +++ b/modoboa_installer/scripts/files/amavis/amavis_postgres_2.11.0.sql @@ -0,0 +1,187 @@ +CREATE TABLE policy ( + id serial PRIMARY KEY, -- 'id' is the _only_ required field + policy_name varchar(32), -- not used by amavisd-new, a comment + + virus_lover char(1) default NULL, -- Y/N + spam_lover char(1) default NULL, -- Y/N + unchecked_lover char(1) default NULL, -- Y/N + banned_files_lover char(1) default NULL, -- Y/N + bad_header_lover char(1) default NULL, -- Y/N + + bypass_virus_checks char(1) default NULL, -- Y/N + bypass_spam_checks char(1) default NULL, -- Y/N + bypass_banned_checks char(1) default NULL, -- Y/N + bypass_header_checks char(1) default NULL, -- Y/N + + virus_quarantine_to varchar(64) default NULL, + spam_quarantine_to varchar(64) default NULL, + banned_quarantine_to varchar(64) default NULL, + unchecked_quarantine_to varchar(64) default NULL, + bad_header_quarantine_to varchar(64) default NULL, + clean_quarantine_to varchar(64) default NULL, + archive_quarantine_to varchar(64) default NULL, + + spam_tag_level real default NULL, -- higher score inserts spam info headers + spam_tag2_level real default NULL, -- inserts 'declared spam' header fields + spam_tag3_level real default NULL, -- inserts 'blatant spam' header fields + spam_kill_level real default NULL, -- higher score triggers evasive actions + -- e.g. reject/drop, quarantine, ... + -- (subject to final_spam_destiny setting) + + spam_dsn_cutoff_level real default NULL, + spam_quarantine_cutoff_level real default NULL, + + addr_extension_virus varchar(64) default NULL, + addr_extension_spam varchar(64) default NULL, + addr_extension_banned varchar(64) default NULL, + addr_extension_bad_header varchar(64) default NULL, + + warnvirusrecip char(1) default NULL, -- Y/N + warnbannedrecip char(1) default NULL, -- Y/N + warnbadhrecip char(1) default NULL, -- Y/N + newvirus_admin varchar(64) default NULL, + virus_admin varchar(64) default NULL, + banned_admin varchar(64) default NULL, + bad_header_admin varchar(64) default NULL, + spam_admin varchar(64) default NULL, + spam_subject_tag varchar(64) default NULL, + spam_subject_tag2 varchar(64) default NULL, + spam_subject_tag3 varchar(64) default NULL, + message_size_limit integer default NULL, -- max size in bytes, 0 disable + banned_rulenames varchar(64) default NULL, -- comma-separated list of ... + -- names mapped through %banned_rules to actual banned_filename tables + disclaimer_options varchar(64) default NULL, + forward_method varchar(64) default NULL, + sa_userconf varchar(64) default NULL, + sa_username varchar(64) default NULL +); + +-- local users +CREATE TABLE users ( + id serial PRIMARY KEY, -- unique id + priority integer NOT NULL DEFAULT 7, -- sort field, 0 is low prior. + policy_id integer NOT NULL DEFAULT 1 CHECK (policy_id >= 0) REFERENCES policy(id), + email bytea NOT NULL UNIQUE, -- email address, non-rfc2822-quoted + fullname varchar(255) DEFAULT NULL -- not used by amavisd-new + -- local char(1) -- Y/N (optional, see SQL section in README.lookups) +); + +-- any e-mail address (non- rfc2822-quoted), external or local, +-- used as senders in wblist +CREATE TABLE mailaddr ( + id serial PRIMARY KEY, + priority integer NOT NULL DEFAULT 9, -- 0 is low priority + email bytea NOT NULL UNIQUE +); + +-- per-recipient whitelist and/or blacklist, +-- puts sender and recipient in relation wb (white or blacklisted sender) +CREATE TABLE wblist ( + rid integer NOT NULL CHECK (rid >= 0) REFERENCES users(id), + sid integer NOT NULL CHECK (sid >= 0) REFERENCES mailaddr(id), + wb varchar(10) NOT NULL, -- W or Y / B or N / space=neutral / score + PRIMARY KEY (rid,sid) +); + +-- grant usage rights: +GRANT select ON policy TO amavis; +GRANT select ON users TO amavis; +GRANT select ON mailaddr TO amavis; +GRANT select ON wblist TO amavis; + + +-- R/W part of the dataset (optional) +-- May reside in the same or in a separate database as lookups database; +-- REQUIRES SUPPORT FOR TRANSACTIONS; specified in @storage_sql_dsn +-- +-- Please create additional indexes on keys when needed, or drop suggested +-- ones as appropriate to optimize queries needed by a management application. +-- See your database documentation for further optimization hints. + +-- provide unique id for each e-mail address, avoids storing copies +CREATE TABLE maddr ( + id serial PRIMARY KEY, + partition_tag integer DEFAULT 0, -- see $partition_tag + email bytea NOT NULL, -- full e-mail address + domain varchar(255) NOT NULL, -- only domain part of the email address + -- with subdomain fields in reverse + CONSTRAINT part_email UNIQUE (partition_tag,email) +); + +-- information pertaining to each processed message as a whole; +-- NOTE: records with a NULL msgs.content should be ignored by utilities, +-- as such records correspond to messages just being processed, or were lost +CREATE TABLE msgs ( + partition_tag integer DEFAULT 0, -- see $partition_tag + mail_id bytea NOT NULL, -- long-term unique mail id, dflt 12 ch + secret_id bytea DEFAULT '', -- authorizes release of mail_id, 12 ch + am_id varchar(20) NOT NULL, -- id used in the log + time_num integer NOT NULL CHECK (time_num >= 0), + -- rx_time: seconds since Unix epoch + time_iso timestamp WITH TIME ZONE NOT NULL,-- rx_time: ISO8601 UTC ascii time + sid integer NOT NULL CHECK (sid >= 0), -- sender: maddr.id + policy varchar(255) DEFAULT '', -- policy bank path (like macro %p) + client_addr varchar(255) DEFAULT '', -- SMTP client IP address (IPv4 or v6) + size integer NOT NULL CHECK (size >= 0), -- message size in bytes + originating char(1) DEFAULT ' ' NOT NULL, -- sender from inside or auth'd + content char(1), -- content type: V/B/U/S/Y/M/H/O/T/C + -- virus/banned/unchecked/spam(kill)/spammy(tag2)/ + -- /bad-mime/bad-header/oversized/mta-err/clean + -- is NULL on partially processed mail + -- (prior to 2.7.0 the CC_SPAMMY was logged as 's', now 'Y' is used; + --- to avoid a need for case-insenstivity in queries) + quar_type char(1), -- quarantined as: ' '/F/Z/B/Q/M/L + -- none/file/zipfile/bsmtp/sql/ + -- /mailbox(smtp)/mailbox(lmtp) + quar_loc varchar(255) DEFAULT '', -- quarantine location (e.g. file) + dsn_sent char(1), -- was DSN sent? Y/N/q (q=quenched) + spam_level real, -- SA spam level (no boosts) + message_id varchar(255) DEFAULT '', -- mail Message-ID header field + from_addr varchar(255) DEFAULT '', -- mail From header field, UTF8 + subject varchar(255) DEFAULT '', -- mail Subject header field, UTF8 + host varchar(255) NOT NULL, -- hostname where amavisd is running + CONSTRAINT msgs_partition_mail UNIQUE (partition_tag,mail_id), + PRIMARY KEY (partition_tag,mail_id) +--FOREIGN KEY (sid) REFERENCES maddr(id) ON DELETE RESTRICT +); +CREATE INDEX msgs_idx_sid ON msgs (sid); +CREATE INDEX msgs_idx_mess_id ON msgs (message_id); -- useful with pen pals +CREATE INDEX msgs_idx_time_iso ON msgs (time_iso); +CREATE INDEX msgs_idx_time_num ON msgs (time_num); -- optional + +-- per-recipient information related to each processed message; +-- NOTE: records in msgrcpt without corresponding msgs.mail_id record are +-- orphaned and should be ignored and eventually deleted by external utilities +CREATE TABLE msgrcpt ( + partition_tag integer DEFAULT 0, -- see $partition_tag + mail_id bytea NOT NULL, -- (must allow duplicates) + rseqnum integer DEFAULT 0 NOT NULL, -- recip's enumeration within msg + rid integer NOT NULL, -- recipient: maddr.id (duplicates allowed) + is_local char(1) DEFAULT ' ' NOT NULL, -- recip is: Y=local, N=foreign + content char(1) DEFAULT ' ' NOT NULL, -- content type V/B/U/S/Y/M/H/O/T/C + ds char(1) NOT NULL, -- delivery status: P/R/B/D/T + -- pass/reject/bounce/discard/tempfail + rs char(1) NOT NULL, -- release status: initialized to ' ' + bl char(1) DEFAULT ' ', -- sender blacklisted by this recip + wl char(1) DEFAULT ' ', -- sender whitelisted by this recip + bspam_level real, -- per-recipient (total) spam level + smtp_resp varchar(255) DEFAULT '', -- SMTP response given to MTA + CONSTRAINT msgrcpt_partition_mail_rseq UNIQUE (partition_tag,mail_id,rseqnum), + PRIMARY KEY (partition_tag,mail_id,rseqnum) +--FOREIGN KEY (rid) REFERENCES maddr(id) ON DELETE RESTRICT, +--FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE +); +CREATE INDEX msgrcpt_idx_mail_id ON msgrcpt (mail_id); +CREATE INDEX msgrcpt_idx_rid ON msgrcpt (rid); + +-- mail quarantine in SQL, enabled by $*_quarantine_method='sql:' +-- NOTE: records in quarantine without corresponding msgs.mail_id record are +-- orphaned and should be ignored and eventually deleted by external utilities +CREATE TABLE quarantine ( + partition_tag integer DEFAULT 0, -- see $partition_tag + mail_id bytea NOT NULL, -- long-term unique mail id + chunk_ind integer NOT NULL CHECK (chunk_ind >= 0), -- chunk number, 1.. + mail_text bytea NOT NULL, -- store mail as chunks of octects + PRIMARY KEY (partition_tag,mail_id,chunk_ind) +--FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE +); From 0da184514ab743f9c69717d26fe0293d76c5c205 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sat, 29 Jul 2017 14:04:11 +0200 Subject: [PATCH 07/56] Added 1.8.1 to compatibility matrix. --- modoboa_installer/compatibility_matrix.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modoboa_installer/compatibility_matrix.py b/modoboa_installer/compatibility_matrix.py index b986e96..ce90bb7 100644 --- a/modoboa_installer/compatibility_matrix.py +++ b/modoboa_installer/compatibility_matrix.py @@ -1,9 +1,10 @@ """Modoboa compatibility matrix.""" COMPATIBILITY_MATRIX = { - # Example: - # - # "1.8.1": {} + "1.8.1": { + "modoboa-webmail": "<=1.1.5", + "modoboa-sievefilers": "<=1.1.1", + } } EXTENSIONS_AVAILABILITY = { From d4d13bd95ee37d39c60bcd92f8871d664c32a19c Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sat, 29 Jul 2017 14:11:51 +0200 Subject: [PATCH 08/56] Fixed compatibility matrix. --- modoboa_installer/compatibility_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/compatibility_matrix.py b/modoboa_installer/compatibility_matrix.py index ce90bb7..dbb92d8 100644 --- a/modoboa_installer/compatibility_matrix.py +++ b/modoboa_installer/compatibility_matrix.py @@ -3,7 +3,7 @@ COMPATIBILITY_MATRIX = { "1.8.1": { "modoboa-webmail": "<=1.1.5", - "modoboa-sievefilers": "<=1.1.1", + "modoboa-sievefilers": "<=1.1.0", } } From 396e520fba9a3e25b7cf1659240941b95529024d Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Mon, 31 Jul 2017 13:50:00 +0200 Subject: [PATCH 09/56] Updated compatibility matrix. --- modoboa_installer/compatibility_matrix.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modoboa_installer/compatibility_matrix.py b/modoboa_installer/compatibility_matrix.py index dbb92d8..e399143 100644 --- a/modoboa_installer/compatibility_matrix.py +++ b/modoboa_installer/compatibility_matrix.py @@ -2,8 +2,9 @@ COMPATIBILITY_MATRIX = { "1.8.1": { - "modoboa-webmail": "<=1.1.5", + "modoboa-pdfcredentials": "<=1.1.0", "modoboa-sievefilers": "<=1.1.0", + "modoboa-webmail": "<=1.1.5", } } From ea8b4e8afa81463b8fa0ffbb1e32aa65513225d6 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Tue, 1 Aug 2017 09:25:36 +0200 Subject: [PATCH 10/56] Fixed typo. --- modoboa_installer/compatibility_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/compatibility_matrix.py b/modoboa_installer/compatibility_matrix.py index e399143..904f4c7 100644 --- a/modoboa_installer/compatibility_matrix.py +++ b/modoboa_installer/compatibility_matrix.py @@ -3,7 +3,7 @@ COMPATIBILITY_MATRIX = { "1.8.1": { "modoboa-pdfcredentials": "<=1.1.0", - "modoboa-sievefilers": "<=1.1.0", + "modoboa-sievefilters": "<=1.1.0", "modoboa-webmail": "<=1.1.5", } } From 1c3c28427e75c855bb19f7f9b13301c1248459d6 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Tue, 1 Aug 2017 18:27:23 +0200 Subject: [PATCH 11/56] Do not override secret key... --- modoboa_installer/scripts/modoboa.py | 8 ++++++++ modoboa_installer/utils.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 6c8361e..e264d24 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -188,6 +188,14 @@ class Modoboa(base.Installer): "handle_mailboxes": True, "account_auto_removal": True }, + # FIXME: since we rewrite all parameters, the secret key + # previously created will disappear. As a quick fix, we + # recreate a new one here but it will mess up opened + # sessions if the installer is used to upgrade an existing + # database... + "core": { + "secret_key": utils.random_key() + }, "modoboa_amavis": { "am_pdp_mode": "inet", }, diff --git a/modoboa_installer/utils.py b/modoboa_installer/utils.py index e4baee8..8402a80 100644 --- a/modoboa_installer/utils.py +++ b/modoboa_installer/utils.py @@ -208,3 +208,17 @@ def convert_version_to_int(version): number += num << total_bits total_bits += bits return number + + +def random_key(l=16): + """Generate a random key. + + :param integer l: the key's length + :return: a string + """ + punctuation = """!#$%&()*+,-./:;<=>?@[]^_`{|}~""" + population = string.digits + string.ascii_letters + punctuation + while True: + key = "".join(random.sample(population * l, l)) + if len(key) == l: + return key From 616c810de2db250c5f8b95f2113d798fb0f2467e Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 9 Aug 2017 15:17:24 +0200 Subject: [PATCH 12/56] Added extra packages. fix #135 --- modoboa_installer/scripts/amavis.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modoboa_installer/scripts/amavis.py b/modoboa_installer/scripts/amavis.py index d33211e..3f8ef7e 100644 --- a/modoboa_installer/scripts/amavis.py +++ b/modoboa_installer/scripts/amavis.py @@ -13,8 +13,15 @@ class Amavis(base.Installer): appname = "amavis" packages = { - "deb": ["libdbi-perl", "amavisd-new"], - "rpm": ["amavisd-new"], + "deb": [ + "libdbi-perl", "amavisd-new", "arc", "arj", "cabextract", + "liblz4-tool", "lrzip", "lzop", "p7zip-full", "rpm2cpio", + "unrar-free", "zoo", "ripole" + ], + "rpm": [ + "amavisd-new", "arj", "cabextract", "lz4", "lrzip", + "lzop", "p7zip", "unar", "unzoo" + ], } with_db = True From c32c0ea2aed431d27f5a234424c32a96fe8bdd42 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 9 Aug 2017 15:18:47 +0200 Subject: [PATCH 13/56] Added autodiscover config. fix #150 --- .../scripts/files/nginx/modoboa.conf.tpl | 1 + modoboa_installer/scripts/nginx.py | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/modoboa_installer/scripts/files/nginx/modoboa.conf.tpl b/modoboa_installer/scripts/files/nginx/modoboa.conf.tpl index ac23191..c4b0f84 100644 --- a/modoboa_installer/scripts/files/nginx/modoboa.conf.tpl +++ b/modoboa_installer/scripts/files/nginx/modoboa.conf.tpl @@ -40,4 +40,5 @@ server { uwsgi_param UWSGI_SCRIPT instance.wsgi:application; uwsgi_pass modoboa; } + %{extra_config} } \ No newline at end of file diff --git a/modoboa_installer/scripts/nginx.py b/modoboa_installer/scripts/nginx.py index f1d5f9f..def4d78 100644 --- a/modoboa_installer/scripts/nginx.py +++ b/modoboa_installer/scripts/nginx.py @@ -29,12 +29,12 @@ class Nginx(base.Installer): }) return context - def _setup_config(self, app, hostname=None): + def _setup_config(self, app, hostname=None, extra_config=None): """Custom app configuration.""" if hostname is None: hostname = self.config.get("general", "hostname") context = self.get_template_context(app) - context.update({"hostname": hostname}) + context.update({"hostname": hostname, "extra_config": extra_config}) src = self.get_file_path("{}.conf.tpl".format(app)) if package.backend.FORMAT == "deb": dst = os.path.join( @@ -57,11 +57,24 @@ class Nginx(base.Installer): def post_run(self): """Additionnal tasks.""" - self._setup_config("modoboa") + extra_modoboa_config = "" if self.config.getboolean("automx", "enabled"): hostname = "autoconfig.{}".format( self.config.get("general", "domain")) self._setup_config("automx", hostname) + extra_modoboa_config = """ + location /autodiscover/autodiscover.xml { + include uwsgi_params; + uwsgi_pass automx; + } + location /mobileconfig { + include uwsgi_params; + uwsgi_pass automx; + } +""" + self._setup_config( + "modoboa", extra_config=extra_modoboa_config) + if not os.path.exists("{}/dhparam.pem".format(self.config_dir)): cmd = "openssl dhparam -dsaparam -out dhparam.pem 4096" utils.exec_cmd(cmd, cwd=self.config_dir) From 01b6ed6095befd3674bc6c0b5017b9c5bb6614a0 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 9 Aug 2017 18:48:24 +0200 Subject: [PATCH 14/56] Added support for debian stretch/mariadb combo. fix #142 --- modoboa_installer/database.py | 44 ++++++++++++++----- .../files/amavis/amavis_mysql_2.11.0.sql | 18 ++++---- .../files/amavis/amavis_postgres_2.10.1.sql | 2 + 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/modoboa_installer/database.py b/modoboa_installer/database.py index 86c4296..9b4edd4 100644 --- a/modoboa_installer/database.py +++ b/modoboa_installer/database.py @@ -1,6 +1,7 @@ """Database related tools.""" import os +import platform import pwd import stat @@ -124,32 +125,51 @@ class MySQL(Database): """MySQL backend.""" packages = { - "deb": ["mysql-server", "libmysqlclient-dev"], + "deb": ["mariadb-server"], "rpm": ["mariadb", "mariadb-devel", "mariadb-server"], } - service = "mariadb" if package.backend.FORMAT == "rpm" else "mysql" + service = "mariadb" + + def _escape(self, query): + """Replace special characters.""" + return query.replace("'", "'\"'\"'") def install_package(self): """Preseed package installation.""" - package.backend.preconfigure( - "mysql-server", "root_password", "password", self.dbpassword) - package.backend.preconfigure( - "mysql-server", "root_password_again", "password", self.dbpassword) + name, version, _id = platform.linux_distribution() + if name == "debian": + mysql_name = "mysql" if version.startswith("8") else "mariadb" + self.packages["deb"].append("lib{}client-dev".format(mysql_name)) super(MySQL, self).install_package() - if package.backend.FORMAT == "rpm": - utils.exec_cmd("mysqladmin -u root password '{}'".format( - self.dbpassword)) + if name == "debian" and version.startswith("8"): + package.backend.preconfigure( + "mariadb-server", "root_password", "password", + self.dbpassword) + package.backend.preconfigure( + "mariadb-server", "root_password_again", "password", + self.dbpassword) + else: + queries = [ + "UPDATE user SET plugin='' WHERE user='root'", + "UPDATE user SET password=PASSWORD('{}') WHERE USER='root'" + .format(self.dbpassword), + "flush privileges" + ] + for query in queries: + utils.exec_cmd( + "mysql -D mysql -e '{}'".format(self._escape(query))) def _exec_query(self, query, dbname=None, dbuser=None, dbpassword=None): """Exec a mysql query.""" if dbuser is None and dbpassword is None: dbuser = self.dbuser dbpassword = self.dbpassword - cmd = "mysql -h {} -u {} -p{}".format(self.dbhost, dbuser, dbpassword) + cmd = "mysql -h {} -u {}".format(self.dbhost, dbuser) + if dbpassword: + cmd += " -p{}".format(dbpassword) if dbname: cmd += " -D {}".format(dbname) - query = query.replace("'", "'\"'\"'") - utils.exec_cmd(cmd + """ -e '{}' """.format(query)) + utils.exec_cmd(cmd + """ -e '{}' """.format(self._escape(query))) def create_user(self, name, password): """Create a user.""" diff --git a/modoboa_installer/scripts/files/amavis/amavis_mysql_2.11.0.sql b/modoboa_installer/scripts/files/amavis/amavis_mysql_2.11.0.sql index 8c9289c..f11792f 100644 --- a/modoboa_installer/scripts/files/amavis/amavis_mysql_2.11.0.sql +++ b/modoboa_installer/scripts/files/amavis/amavis_mysql_2.11.0.sql @@ -107,7 +107,7 @@ CREATE TABLE policy ( -- SELECT COUNT(*) FROM T statement, InnoDB must scan an index of the table, -- which takes some time if the index is not entirely in the buffer pool. -- --- Wayne Smith adds: When using MySQL with InnoDB one might want to +-- Wayne Smith adds: When using MySQL with InnoDB one might want to -- increase buffer size for both pool and log, and might also want -- to change flush settings for a little better performance. Example: -- innodb_buffer_pool_size = 384M @@ -157,22 +157,22 @@ CREATE TABLE msgs ( dsn_sent char(1), -- was DSN sent? Y/N/q (q=quenched) spam_level float, -- SA spam level (no boosts) message_id varchar(255) DEFAULT '', -- mail Message-ID header field - from_addr varchar(255) CHARACTER SET utf8mb4 COLLATE utf8_bin DEFAULT '', + from_addr varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '', -- mail From header field, UTF8 - subject varchar(255) CHARACTER SET utf8mb4 COLLATE utf8_bin DEFAULT '', + subject varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '', -- mail Subject header field, UTF8 host varchar(255) NOT NULL, -- hostname where amavisd is running - PRIMARY KEY (partition_tag,mail_id) + PRIMARY KEY (partition_tag,mail_id), FOREIGN KEY (sid) REFERENCES maddr(id) ON DELETE RESTRICT ) ENGINE=InnoDB; CREATE INDEX msgs_idx_sid ON msgs (sid); CREATE INDEX msgs_idx_mess_id ON msgs (message_id); -- useful with pen pals CREATE INDEX msgs_idx_time_num ON msgs (time_num); -- alternatively when purging based on time_iso (instead of msgs_idx_time_num): --- CREATE INDEX msgs_idx_time_iso ON msgs (time_iso); +CREATE INDEX msgs_idx_time_iso ON msgs (time_iso); -- When using FOREIGN KEY contraints, InnoDB requires index on a field -- (an the field must be the first field in the index). Hence create it: --- CREATE INDEX msgs_idx_mail_id ON msgs (mail_id); +CREATE INDEX msgs_idx_mail_id ON msgs (mail_id); -- per-recipient information related to each processed message; -- NOTE: records in msgrcpt without corresponding msgs.mail_id record are @@ -191,12 +191,14 @@ CREATE TABLE msgrcpt ( wl char(1) DEFAULT ' ', -- sender whitelisted by this recip bspam_level float, -- per-recipient (total) spam level smtp_resp varchar(255) DEFAULT '', -- SMTP response given to MTA - PRIMARY KEY (partition_tag,mail_id,rseqnum) + PRIMARY KEY (partition_tag,mail_id,rseqnum), FOREIGN KEY (rid) REFERENCES maddr(id) ON DELETE RESTRICT, FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE ) ENGINE=InnoDB; CREATE INDEX msgrcpt_idx_mail_id ON msgrcpt (mail_id); CREATE INDEX msgrcpt_idx_rid ON msgrcpt (rid); +-- Additional index on rs since Modoboa uses it to filter its quarantine +CREATE INDEX msgrcpt_idx_rs ON msgrcpt (rs); -- mail quarantine in SQL, enabled by $*_quarantine_method='sql:' -- NOTE: records in quarantine without corresponding msgs.mail_id record are @@ -206,6 +208,6 @@ CREATE TABLE quarantine ( mail_id varbinary(16) NOT NULL, -- long-term unique mail id chunk_ind integer unsigned NOT NULL, -- chunk number, starting with 1 mail_text blob NOT NULL, -- store mail as chunks of octets - PRIMARY KEY (partition_tag,mail_id,chunk_ind) + PRIMARY KEY (partition_tag,mail_id,chunk_ind), FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE ) ENGINE=InnoDB; diff --git a/modoboa_installer/scripts/files/amavis/amavis_postgres_2.10.1.sql b/modoboa_installer/scripts/files/amavis/amavis_postgres_2.10.1.sql index 06cb3b0..99e862e 100644 --- a/modoboa_installer/scripts/files/amavis/amavis_postgres_2.10.1.sql +++ b/modoboa_installer/scripts/files/amavis/amavis_postgres_2.10.1.sql @@ -173,6 +173,8 @@ CREATE TABLE msgrcpt ( ); CREATE INDEX msgrcpt_idx_mail_id ON msgrcpt (mail_id); CREATE INDEX msgrcpt_idx_rid ON msgrcpt (rid); +-- Additional index on rs since Modoboa uses it to filter its quarantine +CREATE INDEX msgrcpt_idx_rs ON msgrcpt (rs); -- mail quarantine in SQL, enabled by $*_quarantine_method='sql:' -- NOTE: records in quarantine without corresponding msgs.mail_id record are From 11a27a66e30557b8c0b966998945564d7163410b Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 9 Aug 2017 18:59:08 +0200 Subject: [PATCH 15/56] Added missing index on rs field. --- .../scripts/files/amavis/amavis_postgres_2.11.0.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modoboa_installer/scripts/files/amavis/amavis_postgres_2.11.0.sql b/modoboa_installer/scripts/files/amavis/amavis_postgres_2.11.0.sql index 06cb3b0..99e862e 100644 --- a/modoboa_installer/scripts/files/amavis/amavis_postgres_2.11.0.sql +++ b/modoboa_installer/scripts/files/amavis/amavis_postgres_2.11.0.sql @@ -173,6 +173,8 @@ CREATE TABLE msgrcpt ( ); CREATE INDEX msgrcpt_idx_mail_id ON msgrcpt (mail_id); CREATE INDEX msgrcpt_idx_rid ON msgrcpt (rid); +-- Additional index on rs since Modoboa uses it to filter its quarantine +CREATE INDEX msgrcpt_idx_rs ON msgrcpt (rs); -- mail quarantine in SQL, enabled by $*_quarantine_method='sql:' -- NOTE: records in quarantine without corresponding msgs.mail_id record are From 6c6945c11b8763851732041e3c2fbd4abda44107 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 11 Aug 2017 16:06:15 +0200 Subject: [PATCH 16/56] Enabled update_statistics cron job. --- modoboa_installer/scripts/files/modoboa/crontab.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/modoboa_installer/scripts/files/modoboa/crontab.tpl b/modoboa_installer/scripts/files/modoboa/crontab.tpl index eaf455e..f58c489 100644 --- a/modoboa_installer/scripts/files/modoboa/crontab.tpl +++ b/modoboa_installer/scripts/files/modoboa/crontab.tpl @@ -21,6 +21,7 @@ INSTANCE=%{instance_path} # Logs parsing */5 * * * * root $PYTHON $INSTANCE/manage.py logparser &> /dev/null +0 * * * * root $PYTHON $INSTANCE/manage.py update_statistics # Radicale rights file %{radicale_enabled}*/2 * * * * root $PYTHON $INSTANCE/manage.py generate_rights From 2b1456603d0626220f091f9e871f9f7c8c28f470 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 11 Aug 2017 16:06:50 +0200 Subject: [PATCH 17/56] Enabled post-login tracking. --- modoboa_installer/scripts/dovecot.py | 7 ++++++- .../conf.d/{10-master.conf => 10-master.conf.tpl} | 11 +++++++++++ .../scripts/files/dovecot/postlogin-mysql.sh.tpl | 7 +++++++ .../scripts/files/dovecot/postlogin-postgres.sh.tpl | 5 +++++ 4 files changed, 29 insertions(+), 1 deletion(-) rename modoboa_installer/scripts/files/dovecot/conf.d/{10-master.conf => 10-master.conf.tpl} (94%) create mode 100644 modoboa_installer/scripts/files/dovecot/postlogin-mysql.sh.tpl create mode 100644 modoboa_installer/scripts/files/dovecot/postlogin-postgres.sh.tpl diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 37b5146..04ce6b4 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -25,7 +25,7 @@ class Dovecot(base.Installer): } config_files = [ "dovecot.conf", "dovecot-dict-sql.conf.ext", "conf.d/10-ssl.conf", - "conf.d/20-lmtp.conf"] + "conf.d/10-master.conf", "conf.d/20-lmtp.conf"] with_user = True def get_config_files(self): @@ -35,6 +35,8 @@ class Dovecot(base.Installer): .format(self.dbengine), "dovecot-sql-master-{}.conf.ext=dovecot-sql-master.conf.ext" .format(self.dbengine), + "postlogin-{}.sh=/usr/local/bin/postlogin.sh" + .format(self.dbengine), ] def get_packages(self): @@ -70,6 +72,7 @@ class Dovecot(base.Installer): "db_driver": self.db_driver, "mailboxes_owner_uid": pw[2], "mailboxes_owner_gid": pw[3], + "modoboa_user": self.config.get("modoboa", "user"), "modoboa_dbname": self.config.get("modoboa", "dbname"), "modoboa_dbuser": self.config.get("modoboa", "dbuser"), "modoboa_dbpassword": self.config.get("modoboa", "dbpassword"), @@ -95,6 +98,8 @@ class Dovecot(base.Installer): ) for f in glob.glob("{}/*".format(self.get_file_path("conf.d"))): utils.copy_file(f, "{}/conf.d".format(self.config_dir)) + # Make postlogin script executable + utils.exec_cmd("chmod +x /usr/local/bin/postlogin.sh") def restart_daemon(self): """Restart daemon process. diff --git a/modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf b/modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf.tpl similarity index 94% rename from modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf rename to modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf.tpl index 6f35a7c..740f169 100644 --- a/modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf +++ b/modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf.tpl @@ -71,11 +71,22 @@ service imap { # Max. number of IMAP processes (connections) #process_limit = 1024 + + executable = imap postlogin } service pop3 { # Max. number of POP3 processes (connections) #process_limit = 1024 + + executable = pop3 postlogin +} + +service postlogin { + executable = script-login /usr/local/bin/postlogin.sh + user = %modoboa_user + unix_listener postlogin { + } } service auth { diff --git a/modoboa_installer/scripts/files/dovecot/postlogin-mysql.sh.tpl b/modoboa_installer/scripts/files/dovecot/postlogin-mysql.sh.tpl new file mode 100644 index 0000000..57da572 --- /dev/null +++ b/modoboa_installer/scripts/files/dovecot/postlogin-mysql.sh.tpl @@ -0,0 +1,7 @@ +#!/bin/sh + +DBNAME=%modoboa_dbname DBUSER=%modoboa_dbuser DBPASSWORD=%modoboa_dbpassword + +echo "UPDATE core_user SET last_login=now() WHERE username='$USER'" | mysql -u $DBUSER -p$DBPASSWORD $DBNAME + +exec "$@" diff --git a/modoboa_installer/scripts/files/dovecot/postlogin-postgres.sh.tpl b/modoboa_installer/scripts/files/dovecot/postlogin-postgres.sh.tpl new file mode 100644 index 0000000..a556ecb --- /dev/null +++ b/modoboa_installer/scripts/files/dovecot/postlogin-postgres.sh.tpl @@ -0,0 +1,5 @@ +#!/bin/sh + +psql -c "UPDATE core_user SET last_login=now() WHERE username='$USER'" > /dev/null + +exec "$@" From 911cd9a0862c23262f04140b1e4b45bb6ab882df Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 8 Sep 2017 10:49:11 +0200 Subject: [PATCH 18/56] Added 1.8.2 to compat. matrix --- modoboa_installer/compatibility_matrix.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modoboa_installer/compatibility_matrix.py b/modoboa_installer/compatibility_matrix.py index 904f4c7..c82e101 100644 --- a/modoboa_installer/compatibility_matrix.py +++ b/modoboa_installer/compatibility_matrix.py @@ -5,6 +5,11 @@ COMPATIBILITY_MATRIX = { "modoboa-pdfcredentials": "<=1.1.0", "modoboa-sievefilters": "<=1.1.0", "modoboa-webmail": "<=1.1.5", + }, + "1.8.2": { + "modoboa-pdfcredentials": ">=1.1.1", + "modoboa-sievefilters": ">=1.1.1", + "modoboa-webmail": ">=1.2.0", } } From 5c7d5240527a2fd11e4b3b012767009a02750f1a Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sat, 9 Sep 2017 09:27:51 +0200 Subject: [PATCH 19/56] Added 1.8.3 to compat. matrix. --- modoboa_installer/compatibility_matrix.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modoboa_installer/compatibility_matrix.py b/modoboa_installer/compatibility_matrix.py index c82e101..17f9ea0 100644 --- a/modoboa_installer/compatibility_matrix.py +++ b/modoboa_installer/compatibility_matrix.py @@ -10,7 +10,12 @@ COMPATIBILITY_MATRIX = { "modoboa-pdfcredentials": ">=1.1.1", "modoboa-sievefilters": ">=1.1.1", "modoboa-webmail": ">=1.2.0", - } + }, + "1.8.3": { + "modoboa-pdfcredentials": ">=1.1.1", + "modoboa-sievefilters": ">=1.1.1", + "modoboa-webmail": ">=1.2.0", + }, } EXTENSIONS_AVAILABILITY = { From 6bcd12b3a7eecb49b948af51d475bd6fb373e406 Mon Sep 17 00:00:00 2001 From: acidsploit Date: Mon, 11 Sep 2017 11:08:36 +0200 Subject: [PATCH 20/56] Fix typo Fix typo which breaks autodiscover See issue #154 --- modoboa_installer/scripts/files/automx/automx.conf.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/files/automx/automx.conf.tpl b/modoboa_installer/scripts/files/automx/automx.conf.tpl index d69a792..61cc835 100644 --- a/modoboa_installer/scripts/files/automx/automx.conf.tpl +++ b/modoboa_installer/scripts/files/automx/automx.conf.tpl @@ -10,7 +10,7 @@ rate_limit_exception_networks = 127.0.0.0/8, ::1/128 [global] backend = sql -actions = settings +action = settings account_type = email host = %sql_dsn query = %sql_query From 19ac9350d774d1b63fb0564bdb25e2fa95ba6b00 Mon Sep 17 00:00:00 2001 From: Yohann Rebattu Date: Sun, 8 Oct 2017 11:29:34 +0200 Subject: [PATCH 21/56] Add configuration wizard (#158) * add --interactive option refs #133 * delete uneeded template as config is a dict now * minor changes after code review --- installer.cfg.template | 103 ------ modoboa_installer/config_dict_template.py | 362 ++++++++++++++++++++++ modoboa_installer/utils.py | 83 ++++- run.py | 5 +- 4 files changed, 438 insertions(+), 115 deletions(-) delete mode 100644 installer.cfg.template create mode 100644 modoboa_installer/config_dict_template.py diff --git a/installer.cfg.template b/installer.cfg.template deleted file mode 100644 index 7e2cfe7..0000000 --- a/installer.cfg.template +++ /dev/null @@ -1,103 +0,0 @@ -[general] -# %(domain)s is the value specified when launching the installer -hostname = mail.%(domain)s - -[certificate] -generate = true -# Choose between self-signed or letsencrypt -type = self-signed - -[letsencrypt] -email = admin@example.com - -[database] -# Select database engine : postgres or mysql -engine = postgres -#engine = mysql -host = 127.0.0.1 -install = true - -[postgres] -user = postgres -password = - -[mysql] -user = root -password = $mysql_password -charset = utf8 -collation = utf8_general_ci - -[modoboa] -user = modoboa -home_dir = /srv/modoboa -venv_path = %(home_dir)s/env -instance_path = %(home_dir)s/instance -timezone = Europe/Paris -dbname = modoboa -dbuser = modoboa -dbpassword = $modoboa_password -# Extensions to install -# also available: modoboa-radicale modoboa-dmarc modoboa-imap-migration -extensions = modoboa-amavis modoboa-pdfcredentials modoboa-postfix-autoreply modoboa-sievefilters modoboa-stats modoboa-webmail modoboa-contacts - -# Deploy Modoboa and enable development mode -devmode = false - -[automx] -enabled = true -user = automx -config_dir = /etc -home_dir = /srv/automx -venv_path = %(home_dir)s/env -instance_path = %(home_dir)s/instance - -[amavis] -enabled = true -user = amavis -max_servers = 1 - -dbname = amavis -dbuser = amavis -dbpassword = $amavis_password - -[clamav] -enabled = true -user = clamav - -[dovecot] -enabled = true -config_dir = /etc/dovecot -user = vmail -home_dir = /srv/vmail -mailboxes_owner = vmail -# Enable extra procotols (in addition to imap and lmtp) -# Example: pop3 -extra_protocols = -# Replace localhost with your domain -postmaster_address = postmaster@localhost - -[nginx] -enabled = true -config_dir = /etc/nginx - -[razor] -enabled = true -config_dir = /etc/razor - -[postfix] -enabled = true -config_dir = /etc/postfix -message_size_limit = 11534336 - -[spamassassin] -enabled = true -config_dir = /etc/mail/spamassassin - -dbname = spamassassin -dbuser = spamassassin -dbpassword = $sa_password - -[uwsgi] -enabled = true -config_dir = /etc/uwsgi -nb_processes = 2 diff --git a/modoboa_installer/config_dict_template.py b/modoboa_installer/config_dict_template.py new file mode 100644 index 0000000..55a1319 --- /dev/null +++ b/modoboa_installer/config_dict_template.py @@ -0,0 +1,362 @@ +import random +import string + + +def make_password(length=16): + """Create a random password.""" + return "".join( + random.SystemRandom().choice( + string.ascii_letters + string.digits) for _ in range(length)) + + +# Validators should return a tuple bool, error message +def is_email(user_input): + """Return True in input is a valid email""" + return "@" in user_input, "Please enter a valid email" + + +ConfigDictTemplate = [ + { + "name": "general", + "values": [ + { + "option": "hostname", + "default": "mail.%(domain)s", + } + ] + }, + { + "name": "certificate", + "values": [ + { + "option": "generate", + "default": "true", + }, + { + "option": "type", + "default": "self-signed", + "customizable": True, + "question": "Please choose your certificate type", + "values": ["self-signed", "letsencrypt"], + } + ], + }, + { + "name": "letsencrypt", + "if": "certificate.type=letsencrypt", + "values": [ + { + "option": "email", + "default": "admin@example.com", + "question": ( + "Please enter the mail you wish to use for " + "letsencrypt"), + "customizable": True, + "validators": [is_email] + } + ] + }, + { + "name": "database", + "values": [ + { + "option": "engine", + "default": "postgres", + "customizable": True, + "question": "Please choose your database engine", + "values": ["postgres", "mysql"], + }, + { + "option": "host", + "default": "127.0.0.1", + }, + { + "option": "install", + "default": "true", + } + ] + }, + { + "name": "postgres", + "if": "database.engine=postgres", + "values": [ + { + "option": "user", + "default": "postgres", + }, + { + "option": "password", + "default": "", + "customizable": True, + "question": "Please enter postgres password", + }, + ] + }, + { + "name": "mysql", + "if": "database.engine=mysql", + "values": [ + { + "option": "user", + "default": "root", + }, + { + "option": "password", + "default": make_password, + "customizable": True, + "question": "Please enter mysql root password" + }, + { + "option": "charset", + "default": "utf8", + }, + { + "option": "collation", + "default": "utf8_general_ci", + } + ] + }, + { + "name": "modoboa", + "values": [ + { + "option": "user", + "default": "modoboa", + }, + { + "option": "home_dir", + "default": "/srv/modoboa", + }, + { + "option": "venv_path", + "default": "%(home_dir)s/instance", + }, + { + "option": "instance_path", + "default": "%(home_dir)s/env", + }, + { + "option": "timezone", + "default": "Europe/Paris", + }, + { + "option": "dbname", + "default": "modoboa", + }, + { + "option": "dbuser", + "default": "modoboa", + }, + { + "option": "dbpassword", + "default": make_password, + "customizable": True, + "question": "Please enter Modoboa db password", + }, + { + "option": "extensions", + "default": ( + "modoboa-amavis modoboa-pdfcredentials " + "modoboa-postfix-autoreply modoboa-sievefilters " + "modoboa-stats modoboa-webmail modoboa-contacts"), + }, + { + "option": "devmod", + "default": "false", + }, + ] + }, + { + "name": "automx", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "user", + "default": "automx", + }, + { + "option": "config_dir", + "default": "/etc", + }, + { + "option": "home_dir", + "default": "/srv/automx", + }, + { + "option": "venv_path", + "default": "%(home_dir)s/env", + }, + { + "option": "instance_path", + "default": "%(home_dir)s/instance", + }, + ] + }, + { + "name": "amavis", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "user", + "default": "amavis", + }, + { + "option": "max_servers", + "default": "1", + }, + { + "option": "dbname", + "default": "amavis", + }, + { + "option": "dbuser", + "default": "amavis", + }, + { + "option": "dbpassword", + "default": make_password, + "customizable": True, + "question": "Please enter amavis db password" + }, + ], + }, + { + "name": "clamav", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "user", + "default": "clamav", + }, + ] + }, + { + "name": "dovecot", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/dovecot", + }, + { + "option": "user", + "default": "vmail", + }, + { + "option": "home_dir", + "default": "/srv/vmail", + }, + { + "option": "mailboxes_owner", + "default": "vmail", + }, + { + "option": "extra_protocols", + "default": "", + }, + { + "option": "postmaster_address", + "default": "postmaster@%(domain)s", + }, + ] + }, + { + "name": "nginx", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/nginx", + }, + ], + }, + { + "name": "razor", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/razor", + }, + ] + }, + { + "name": "postfix", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/postfix", + }, + { + "option": "message_size_limit", + "default": "11534336", + }, + ] + }, + { + "name": "spamassassin", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/mail/spamassassin", + }, + { + "option": "dbname", + "default": "spamassassin", + }, + { + "option": "dbuser", + "default": "spamassassin", + }, + { + "option": "dbpassword", + "default": make_password, + "customizable": True, + "question": "Please enter spamassassin db password" + }, + ] + }, + { + "name": "uwsgi", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/uwsgi", + }, + { + "option": "nb_processes", + "default": "2", + }, + ] + }, +] diff --git a/modoboa_installer/utils.py b/modoboa_installer/utils.py index 8402a80..5368521 100644 --- a/modoboa_installer/utils.py +++ b/modoboa_installer/utils.py @@ -10,6 +10,12 @@ import shutil import string import subprocess import sys +try: + import configparser +except ImportError: + import ConfigParser as configparser + +from . import config_dict_template ENV = {} @@ -139,23 +145,14 @@ def copy_from_template(template, dest, context): fp.write(ConfigFileTemplate(buf).substitute(context)) -def check_config_file(dest): +def check_config_file(dest, interactive=False): """Create a new installer config file if needed.""" if os.path.exists(dest): return printcolor( "Configuration file {} not found, creating new one." .format(dest), YELLOW) - with open("installer.cfg.template") as fp: - buf = fp.read() - context = { - "mysql_password": make_password(), - "modoboa_password": make_password(), - "amavis_password": make_password(), - "sa_password": make_password() - } - with open(dest, "w") as fp: - fp.write(string.Template(buf).substitute(context)) + gen_config(dest, interactive) def has_colours(stream): @@ -222,3 +219,67 @@ def random_key(l=16): key = "".join(random.sample(population * l, l)) if len(key) == l: return key + + +def validate(value, config_entry): + if value is None: + return False + if "values" not in config_entry and "validators" not in config_entry: + return True + if "values" in config_entry: + try: + value = int(value) + except ValueError: + return False + return value >= 0 and value < len(config_entry["values"]) + if "validators" in config_entry: + for validator in config_entry["validators"]: + valide, message = validator(value) + if not valide: + printcolor(message, MAGENTA) + return False + return True + + +def get_entry_value(entry, interactive): + if callable(entry["default"]): + default_value = entry["default"]() + else: + default_value = entry["default"] + user_value = None + if entry.get("customizable") and interactive: + while (user_value != '' and not validate(user_value, entry)): + print(entry.get("question")) + if entry.get("values"): + print("Please choose from the list") + values = entry.get("values") + for index, value in enumerate(values): + print("{} {}".format(index, value)) + print("default is <{}>".format(default_value)) + user_value = user_input("->") + + if entry.get("values") and user_value != '': + user_value = values[int(user_value)] + return user_value if user_value else default_value + + +def gen_config(dest, interactive=False): + """Create config file from dict template""" + tpl_dict = config_dict_template.ConfigDictTemplate + config = configparser.ConfigParser() + # only ask about options we need, else still generate default + for section in tpl_dict: + if "if" in section: + config_key, value = section.get("if").split("=") + section_name, option = config_key.split(".") + interactive_section = ( + config.get(section_name, option) == value and interactive) + else: + interactive_section = interactive + config.add_section(section["name"]) + for config_entry in section["values"]: + value = get_entry_value(config_entry, interactive_section) + config.set(section["name"], config_entry["option"], value) + + with open(dest, "w") as configfile: + config.write(configfile) diff --git a/run.py b/run.py index 385829e..0e95ed5 100755 --- a/run.py +++ b/run.py @@ -31,6 +31,9 @@ def main(): parser.add_argument( "--stop-after-configfile-check", action="store_true", default=False, help="Check configuration, generate it if needed and exit") + parser.add_argument( + "--interactive", action="store_true", default=False, + help="Generate configuration file with user interaction") parser.add_argument("domain", type=str, help="The main domain of your future mail server") args = parser.parse_args() @@ -38,7 +41,7 @@ def main(): if args.debug: utils.ENV["debug"] = True utils.printcolor("Welcome to Modoboa installer!", utils.GREEN) - utils.check_config_file(args.configfile) + utils.check_config_file(args.configfile, args.interactive) if args.stop_after_configfile_check: return config = configparser.SafeConfigParser() From 7dcfc6c1282793ff3fe673878511e8f07fa9821d Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 8 Oct 2017 14:38:29 +0200 Subject: [PATCH 22/56] Added unit test. --- .travis.yml | 15 +++++++++++++++ tests.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 .travis.yml create mode 100644 tests.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..83b4593 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,15 @@ +sudo: false +language: python +cache: pip +python: + - "2.7" + - "3.4" + +before_install: + - pip install codecov + +script: + - coverage run tests.py + +after_success: + - codecov diff --git a/tests.py b/tests.py new file mode 100644 index 0000000..b7d9845 --- /dev/null +++ b/tests.py @@ -0,0 +1,33 @@ +"""Installer unit tests.""" + +import os +import shutil +import tempfile +import unittest + +import run + + +class ConfigFileTestCase(unittest.TestCase): + """Test configuration file generation.""" + + def setUp(self): + """Create temp dir.""" + self.workdir = tempfile.mkdtemp() + self.cfgfile = os.path.join(self.workdir, "installer.cfg") + + def tearDown(self): + """Delete temp dir.""" + shutil.rmtree(self.workdir) + + def test_configfile_generation(self): + """Check simple case.""" + run.main([ + "--stop-after-configfile-check", + "--configfile", self.cfgfile, + "example.test"]) + self.assertTrue(os.path.exists(self.cfgfile)) + + +if __name__ == "__main__": + unittest.main() From d94957b79f9413cb5925c81a7861a3e1a04891e1 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 8 Oct 2017 14:39:40 +0200 Subject: [PATCH 23/56] Fixed unit test. --- run.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index 0e95ed5..bfa4b5f 100755 --- a/run.py +++ b/run.py @@ -7,6 +7,7 @@ try: import configparser except ImportError: import ConfigParser as configparser +import sys from modoboa_installer import compatibility_matrix from modoboa_installer import package @@ -15,7 +16,7 @@ from modoboa_installer import ssl from modoboa_installer import utils -def main(): +def main(input_args): """Install process.""" parser = argparse.ArgumentParser() parser.add_argument("--debug", action="store_true", default=False, @@ -36,7 +37,7 @@ def main(): help="Generate configuration file with user interaction") parser.add_argument("domain", type=str, help="The main domain of your future mail server") - args = parser.parse_args() + args = parser.parse_args(input_args) if args.debug: utils.ENV["debug"] = True @@ -91,4 +92,4 @@ def main(): if __name__ == "__main__": - main() + main(sys.argv[1:]) From 93f5fefe6f81d6bf2b38bdcc3feccc9d493895f6 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 8 Oct 2017 14:44:26 +0200 Subject: [PATCH 24/56] Fixed python3 compat. --- README.rst | 5 +++++ run.py | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 91df716..bfc8004 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,8 @@ modoboa-installer ================= +|travis| + An installer which deploy a complete mail server based on Modoboa. .. warning:: @@ -81,3 +83,6 @@ modify the following settings:: Change the ``email`` setting to a valid value since it will be used for account recovery. + +.. |travis| image:: https://travis-ci.org/modoboa/modoboa-installer.png?branch=master + :target: https://travis-ci.org/modoboa/modoboa-installer diff --git a/run.py b/run.py index bfa4b5f..45c666e 100755 --- a/run.py +++ b/run.py @@ -19,6 +19,9 @@ from modoboa_installer import utils def main(input_args): """Install process.""" parser = argparse.ArgumentParser() + versions = ( + ["latest"] + list(compatibility_matrix.COMPATIBILITY_MATRIX.keys()) + ) parser.add_argument("--debug", action="store_true", default=False, help="Enable debug output") parser.add_argument("--force", action="store_true", default=False, @@ -26,8 +29,7 @@ def main(input_args): parser.add_argument("--configfile", default="installer.cfg", help="Configuration file to use") parser.add_argument( - "--version", default="latest", - choices=["latest"] + compatibility_matrix.COMPATIBILITY_MATRIX.keys(), + "--version", default="latest", choices=versions, help="Modoboa version to install") parser.add_argument( "--stop-after-configfile-check", action="store_true", default=False, From fe54e990b51b9120f003cb12615a15e12ec45289 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 8 Oct 2017 15:00:07 +0200 Subject: [PATCH 25/56] Added codecov badge. --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index bfc8004..74e5f61 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ modoboa-installer ================= -|travis| +|travis| |codecov| An installer which deploy a complete mail server based on Modoboa. @@ -86,3 +86,5 @@ for account recovery. .. |travis| image:: https://travis-ci.org/modoboa/modoboa-installer.png?branch=master :target: https://travis-ci.org/modoboa/modoboa-installer +.. |codecov| image:: http://codecov.io/github/modoboa/modoboa-installer/coverage.svg?branch=master + :target: http://codecov.io/github/modoboa/modoboa-installer?branch=master From 7c0346d281ae02731d2436ca4d33b31ea7141ad3 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 8 Oct 2017 15:26:59 +0200 Subject: [PATCH 26/56] Added unit test. --- .travis.yml | 2 +- modoboa_installer/utils.py | 11 ++++++----- test-requirements.txt | 2 ++ tests.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 test-requirements.txt diff --git a/.travis.yml b/.travis.yml index 83b4593..641df12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - "3.4" before_install: - - pip install codecov + - pip install -r test-requirements.txt script: - coverage run tests.py diff --git a/modoboa_installer/utils.py b/modoboa_installer/utils.py index 5368521..5cd2897 100644 --- a/modoboa_installer/utils.py +++ b/modoboa_installer/utils.py @@ -249,16 +249,17 @@ def get_entry_value(entry, interactive): user_value = None if entry.get("customizable") and interactive: while (user_value != '' and not validate(user_value, entry)): - print(entry.get("question")) + question = entry.get("question") if entry.get("values"): - print("Please choose from the list") + question += " from the list" values = entry.get("values") for index, value in enumerate(values): - print("{} {}".format(index, value)) + question += "\n{} {}".format(index, value) + print(question) print("default is <{}>".format(default_value)) - user_value = user_input("->") + user_value = user_input("-> ") - if entry.get("values") and user_value != '': + if entry.get("values") and user_value != "": user_value = values[int(user_value)] return user_value if user_value else default_value diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..6ec2cdd --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,2 @@ +codecov +mock diff --git a/tests.py b/tests.py index b7d9845..247ab47 100644 --- a/tests.py +++ b/tests.py @@ -1,10 +1,20 @@ """Installer unit tests.""" +try: + import configparser +except ImportError: + import ConfigParser as configparser import os import shutil +import sys import tempfile import unittest +try: + from unittest.mock import patch +except ImportError: + from mock import patch + import run @@ -28,6 +38,25 @@ class ConfigFileTestCase(unittest.TestCase): "example.test"]) self.assertTrue(os.path.exists(self.cfgfile)) + @patch("modoboa_installer.utils.user_input") + def test_interactive_mode(self, mock_user_input): + """Check interactive mode.""" + mock_user_input.side_effect = [ + "0", "0", "", "", "", "" + ] + with open(os.devnull, "w") as fp: + sys.stdout = fp + run.main([ + "--stop-after-configfile-check", + "--configfile", self.cfgfile, + "--interactive", + "example.test"]) + self.assertTrue(os.path.exists(self.cfgfile)) + config = configparser.ConfigParser() + config.read(self.cfgfile) + self.assertEqual(config.get("certificate", "type"), "self-signed") + self.assertEqual(config.get("database", "engine"), "postgres") + if __name__ == "__main__": unittest.main() From b7c9b77f8ae0f9f70397e1b266d5d01d99800c2f Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 8 Oct 2017 15:35:00 +0200 Subject: [PATCH 27/56] Added unit test --- tests.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests.py b/tests.py index 247ab47..4db0418 100644 --- a/tests.py +++ b/tests.py @@ -57,6 +57,26 @@ class ConfigFileTestCase(unittest.TestCase): self.assertEqual(config.get("certificate", "type"), "self-signed") self.assertEqual(config.get("database", "engine"), "postgres") + @patch("modoboa_installer.utils.user_input") + def test_interactive_mode_letsencrypt(self, mock_user_input): + """Check interactive mode.""" + mock_user_input.side_effect = [ + "1", "admin@example.test", "0", "", "", "", "" + ] + with open(os.devnull, "w") as fp: + sys.stdout = fp + run.main([ + "--stop-after-configfile-check", + "--configfile", self.cfgfile, + "--interactive", + "example.test"]) + self.assertTrue(os.path.exists(self.cfgfile)) + config = configparser.ConfigParser() + config.read(self.cfgfile) + self.assertEqual(config.get("certificate", "type"), "letsencrypt") + self.assertEqual( + config.get("letsencrypt", "email"), "admin@example.test") + if __name__ == "__main__": unittest.main() From 11447d062a460e8250d14ac53b89b6393a830599 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 8 Oct 2017 15:47:50 +0200 Subject: [PATCH 28/56] Added unit tests. --- test-requirements.txt | 1 + tests.py | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6ec2cdd..55e3fa0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,3 @@ codecov mock +six diff --git a/tests.py b/tests.py index 4db0418..7b86310 100644 --- a/tests.py +++ b/tests.py @@ -1,15 +1,13 @@ """Installer unit tests.""" -try: - import configparser -except ImportError: - import ConfigParser as configparser import os import shutil import sys import tempfile import unittest +from six import StringIO +from six.moves import configparser try: from unittest.mock import patch except ImportError: @@ -77,6 +75,22 @@ class ConfigFileTestCase(unittest.TestCase): self.assertEqual( config.get("letsencrypt", "email"), "admin@example.test") + @patch("modoboa_installer.utils.user_input") + def test_configfile_loading(self, mock_user_input): + """Check interactive mode.""" + mock_user_input.side_effect = ["no"] + out = StringIO() + sys.stdout = out + run.main([ + "--configfile", self.cfgfile, + "example.test"]) + self.assertTrue(os.path.exists(self.cfgfile)) + self.assertIn( + "modoboa automx amavis clamav dovecot nginx razor postfix" + " spamassassin uwsgi", + out.getvalue() + ) + if __name__ == "__main__": unittest.main() From c894013d6789757a643e7ae249ff5cdea4754c86 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Tue, 10 Oct 2017 09:40:30 +0200 Subject: [PATCH 29/56] Fixed default values. see #160 --- README.rst | 4 ++++ modoboa_installer/config_dict_template.py | 6 +++--- run.py | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 74e5f61..f24c596 100644 --- a/README.rst +++ b/README.rst @@ -47,6 +47,10 @@ run the following command:: $ ./run.py --stop-after-configfile-check +An interactive mode is also available:: + + $ ./run.py --interactive + Make your modifications and run the installer as usual. By default, the latest Modoboa version is installed but you can select diff --git a/modoboa_installer/config_dict_template.py b/modoboa_installer/config_dict_template.py index 55a1319..933fae5 100644 --- a/modoboa_installer/config_dict_template.py +++ b/modoboa_installer/config_dict_template.py @@ -129,11 +129,11 @@ ConfigDictTemplate = [ }, { "option": "venv_path", - "default": "%(home_dir)s/instance", + "default": "%(home_dir)s/env", }, { "option": "instance_path", - "default": "%(home_dir)s/env", + "default": "%(home_dir)s/instance", }, { "option": "timezone", @@ -161,7 +161,7 @@ ConfigDictTemplate = [ "modoboa-stats modoboa-webmail modoboa-contacts"), }, { - "option": "devmod", + "option": "devmode", "default": "false", }, ] diff --git a/run.py b/run.py index 45c666e..385da45 100755 --- a/run.py +++ b/run.py @@ -53,6 +53,7 @@ def main(input_args): if not config.has_section("general"): config.add_section("general") config.set("general", "domain", args.domain) + config.set("dovecot", "domain", args.domain) config.set("modoboa", "version", args.version) utils.printcolor( "Your mail server will be installed with the following components:", From 08ba4a4de5ec210486a2f0381ca8c1e1d8519c58 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 13 Oct 2017 18:01:29 +0200 Subject: [PATCH 30/56] Fixed centos issue with uwsgi. --- modoboa_installer/scripts/files/modoboa/sudoers.tpl | 2 +- modoboa_installer/scripts/modoboa.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/files/modoboa/sudoers.tpl b/modoboa_installer/scripts/files/modoboa/sudoers.tpl index 450cbca..22a13f0 100644 --- a/modoboa_installer/scripts/files/modoboa/sudoers.tpl +++ b/modoboa_installer/scripts/files/modoboa/sudoers.tpl @@ -1 +1 @@ -%{user} ALL=(%{dovecot_mailboxes_owner}) NOPASSWD: /usr/bin/doveadm +%{sudo_user} ALL=(%{dovecot_mailboxes_owner}) NOPASSWD: /usr/bin/doveadm diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index e264d24..2a240c7 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -168,6 +168,9 @@ class Modoboa(base.Installer): extensions = self.config.get("modoboa", "extensions") extensions = extensions.split() context.update({ + "sudo_user": ( + "uwsgi" if package.backend.FORMAT == "rpm" else context["user"] + ), "dovecot_mailboxes_owner": ( self.config.get("dovecot", "mailboxes_owner")), "radicale_enabled": "" if "modoboa-radicale" in extensions else "#" From 996575c4dc97225881af667e85e4b2e5dd227dc6 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sat, 14 Oct 2017 09:54:51 +0200 Subject: [PATCH 31/56] Added 1.9.0 to compatibility matrix. --- modoboa_installer/compatibility_matrix.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modoboa_installer/compatibility_matrix.py b/modoboa_installer/compatibility_matrix.py index 17f9ea0..285ae90 100644 --- a/modoboa_installer/compatibility_matrix.py +++ b/modoboa_installer/compatibility_matrix.py @@ -16,6 +16,11 @@ COMPATIBILITY_MATRIX = { "modoboa-sievefilters": ">=1.1.1", "modoboa-webmail": ">=1.2.0", }, + "1.9.0": { + "modoboa-pdfcredentials": ">=1.1.1", + "modoboa-sievefilters": ">=1.1.1", + "modoboa-webmail": ">=1.2.0", + } } EXTENSIONS_AVAILABILITY = { From 8fb527c1d8e023b423041d67bfdaeeda4ea403f5 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Tue, 17 Oct 2017 09:07:46 +0200 Subject: [PATCH 32/56] Added dbuser. (#163) see #161 --- modoboa_installer/scripts/modoboa.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 2a240c7..fa7046c 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -119,9 +119,10 @@ class Modoboa(base.Installer): "--domain", self.config.get("general", "hostname"), "--extensions", " ".join(self.extensions), "--dont-install-extensions", - "--dburl", "'default:{0}://{1}:{2}@{3}/{1}'".format( - self.config.get("database", "engine"), self.dbname, - self.dbpasswd, self.dbhost) + "--dburl", "'default:{}://{}:{}@{}/{}'".format( + self.config.get("database", "engine"), + self.dbuser, self.dbpasswd, self.dbhost, self.dbname + ) ] if self.devmode: args = ["--devel"] + args From 2739228cd756e3072af10e66b22c58b5398bf063 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 27 Oct 2017 16:12:46 +0200 Subject: [PATCH 33/56] Update PATH. fix #159 --- .../scripts/files/dovecot/postlogin-postgres.sh.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modoboa_installer/scripts/files/dovecot/postlogin-postgres.sh.tpl b/modoboa_installer/scripts/files/dovecot/postlogin-postgres.sh.tpl index a556ecb..e91ac2d 100644 --- a/modoboa_installer/scripts/files/dovecot/postlogin-postgres.sh.tpl +++ b/modoboa_installer/scripts/files/dovecot/postlogin-postgres.sh.tpl @@ -1,5 +1,7 @@ #!/bin/sh +PATH="/usr/bin:/usr/local/bin:/bin" + psql -c "UPDATE core_user SET last_login=now() WHERE username='$USER'" > /dev/null exec "$@" From d79d4b13dcaa661d6a1b2153d504586ef746b01b Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 10 Nov 2017 17:27:00 +0100 Subject: [PATCH 34/56] Removed useless lines. see #167 --- modoboa_installer/scripts/files/spamassassin/v310.pre.tpl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modoboa_installer/scripts/files/spamassassin/v310.pre.tpl b/modoboa_installer/scripts/files/spamassassin/v310.pre.tpl index 9f0efef..481d43c 100644 --- a/modoboa_installer/scripts/files/spamassassin/v310.pre.tpl +++ b/modoboa_installer/scripts/files/spamassassin/v310.pre.tpl @@ -75,7 +75,3 @@ loadplugin Mail::SpamAssassin::Plugin::MIMEHeader # ReplaceTags # loadplugin Mail::SpamAssassin::Plugin::ReplaceTags - -# DCC - perform DCC message checks. -# -loadplugin Mail::SpamAssassin::Plugin::DCC From 18f1c85bcfedd3877b65d3f3de74016bb6831585 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 3 Dec 2017 13:41:38 +0100 Subject: [PATCH 35/56] Added message about required DNS records. fix #168 --- run.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 385da45..80c586f 100755 --- a/run.py +++ b/run.py @@ -43,7 +43,7 @@ def main(input_args): if args.debug: utils.ENV["debug"] = True - utils.printcolor("Welcome to Modoboa installer!", utils.GREEN) + utils.printcolor("Welcome to Modoboa installer!\n", utils.GREEN) utils.check_config_file(args.configfile, args.interactive) if args.stop_after_configfile_check: return @@ -55,6 +55,15 @@ def main(input_args): config.set("general", "domain", args.domain) config.set("dovecot", "domain", args.domain) config.set("modoboa", "version", args.version) + utils.printcolor( + "Warning:\n" + "Before you start the installation, please make sure the following " + "DNS records exist for domain '{}':\n" + " mail IN A \n" + " IN MX {}.\n".format( + args.domain, config.get("general", "hostname")), + utils.CYAN + ) utils.printcolor( "Your mail server will be installed with the following components:", utils.BLUE) From 111cb83b07270032a928425009522d94b0c82b88 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sun, 3 Dec 2017 13:48:14 +0100 Subject: [PATCH 36/56] Set mydestination setting to $myhostname. --- modoboa_installer/scripts/files/postfix/main.cf.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/files/postfix/main.cf.tpl b/modoboa_installer/scripts/files/postfix/main.cf.tpl index 2d2d0f6..fc87268 100644 --- a/modoboa_installer/scripts/files/postfix/main.cf.tpl +++ b/modoboa_installer/scripts/files/postfix/main.cf.tpl @@ -2,7 +2,7 @@ inet_interfaces = all inet_protocols = ipv4 myhostname = %hostname myorigin = $myhostname -mydestination = +mydestination = $myhostname mynetworks = 127.0.0.0/8 smtpd_banner = $myhostname ESMTP biff = no From 2c206a52b14631abf5cdbff35186f1f6ece67ac8 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Thu, 7 Dec 2017 11:46:01 +0100 Subject: [PATCH 37/56] Fixed dovecot user query. --- .../scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl | 2 +- .../scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl b/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl index 99eacf5..4af0d86 100644 --- a/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl +++ b/modoboa_installer/scripts/files/dovecot/dovecot-sql-mysql.conf.ext.tpl @@ -123,7 +123,7 @@ connect = host=%dbhost dbname=%modoboa_dbname user=%modoboa_dbuser password=%mod #user_query = \ # SELECT home, uid, gid \ # FROM users WHERE username = '%%n' AND domain = '%%d' -user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' AND u.is_active=1 AND dom.enabled=1 +user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, CONCAT('*:bytes=', mb.quota, 'M') AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' # If you wish to avoid two SQL lookups (passdb + userdb), you can use # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll diff --git a/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl b/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl index 2610750..e6328a5 100644 --- a/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl +++ b/modoboa_installer/scripts/files/dovecot/dovecot-sql-postgres.conf.ext.tpl @@ -123,7 +123,7 @@ connect = host=%dbhost dbname=%modoboa_dbname user=%modoboa_dbuser password=%mod #user_query = \ # SELECT home, uid, gid \ # FROM users WHERE username = '%%n' AND domain = '%%d' -user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' AND u.is_active AND dom.enabled +user_query = SELECT '%{home_dir}/%%d/%%n' AS home, %mailboxes_owner_uid as uid, %mailboxes_owner_gid as gid, '*:bytes=' || mb.quota || 'M' AS quota_rule FROM admin_mailbox mb INNER JOIN admin_domain dom ON mb.domain_id=dom.id INNER JOIN core_user u ON u.id=mb.user_id WHERE mb.address='%%n' AND dom.name='%%d' # If you wish to avoid two SQL lookups (passdb + userdb), you can use # userdb prefetch instead of userdb sql in dovecot.conf. In that case you'll From 0ceea13d4b10b83a7db9d1d66849b9805eee98f5 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Thu, 7 Dec 2017 17:42:38 +0100 Subject: [PATCH 38/56] Added postwhite support. (#171) * Added postwhite support. see #109 * Fixed unit test. --- modoboa_installer/config_dict_template.py | 699 +++++++++--------- modoboa_installer/scripts/base.py | 4 +- .../scripts/files/postfix/main.cf.tpl | 1 + .../scripts/files/postwhite/crontab.tpl | 9 + modoboa_installer/scripts/postfix.py | 4 + modoboa_installer/scripts/postwhite.py | 51 ++ tests.py | 2 +- 7 files changed, 424 insertions(+), 346 deletions(-) create mode 100644 modoboa_installer/scripts/files/postwhite/crontab.tpl create mode 100644 modoboa_installer/scripts/postwhite.py diff --git a/modoboa_installer/config_dict_template.py b/modoboa_installer/config_dict_template.py index 933fae5..6f39dc9 100644 --- a/modoboa_installer/config_dict_template.py +++ b/modoboa_installer/config_dict_template.py @@ -16,347 +16,360 @@ def is_email(user_input): ConfigDictTemplate = [ - { - "name": "general", - "values": [ - { - "option": "hostname", - "default": "mail.%(domain)s", - } - ] - }, - { - "name": "certificate", - "values": [ - { - "option": "generate", - "default": "true", - }, - { - "option": "type", - "default": "self-signed", - "customizable": True, - "question": "Please choose your certificate type", - "values": ["self-signed", "letsencrypt"], - } - ], - }, - { - "name": "letsencrypt", - "if": "certificate.type=letsencrypt", - "values": [ - { - "option": "email", - "default": "admin@example.com", - "question": ( - "Please enter the mail you wish to use for " - "letsencrypt"), - "customizable": True, - "validators": [is_email] - } - ] - }, - { - "name": "database", - "values": [ - { - "option": "engine", - "default": "postgres", - "customizable": True, - "question": "Please choose your database engine", - "values": ["postgres", "mysql"], - }, - { - "option": "host", - "default": "127.0.0.1", - }, - { - "option": "install", - "default": "true", - } - ] - }, - { - "name": "postgres", - "if": "database.engine=postgres", - "values": [ - { - "option": "user", - "default": "postgres", - }, - { - "option": "password", - "default": "", - "customizable": True, - "question": "Please enter postgres password", - }, - ] - }, - { - "name": "mysql", - "if": "database.engine=mysql", - "values": [ - { - "option": "user", - "default": "root", - }, - { - "option": "password", - "default": make_password, - "customizable": True, - "question": "Please enter mysql root password" - }, - { - "option": "charset", - "default": "utf8", - }, - { - "option": "collation", - "default": "utf8_general_ci", - } - ] - }, - { - "name": "modoboa", - "values": [ - { - "option": "user", - "default": "modoboa", - }, - { - "option": "home_dir", - "default": "/srv/modoboa", - }, - { - "option": "venv_path", - "default": "%(home_dir)s/env", - }, - { - "option": "instance_path", - "default": "%(home_dir)s/instance", - }, - { - "option": "timezone", - "default": "Europe/Paris", - }, - { - "option": "dbname", - "default": "modoboa", - }, - { - "option": "dbuser", - "default": "modoboa", - }, - { - "option": "dbpassword", - "default": make_password, - "customizable": True, - "question": "Please enter Modoboa db password", - }, - { - "option": "extensions", - "default": ( - "modoboa-amavis modoboa-pdfcredentials " - "modoboa-postfix-autoreply modoboa-sievefilters " - "modoboa-stats modoboa-webmail modoboa-contacts"), - }, - { - "option": "devmode", - "default": "false", - }, - ] - }, - { - "name": "automx", - "values": [ - { - "option": "enabled", - "default": "true", - }, - { - "option": "user", - "default": "automx", - }, - { - "option": "config_dir", - "default": "/etc", - }, - { - "option": "home_dir", - "default": "/srv/automx", - }, - { - "option": "venv_path", - "default": "%(home_dir)s/env", - }, - { - "option": "instance_path", - "default": "%(home_dir)s/instance", - }, - ] - }, - { - "name": "amavis", - "values": [ - { - "option": "enabled", - "default": "true", - }, - { - "option": "user", - "default": "amavis", - }, - { - "option": "max_servers", - "default": "1", - }, - { - "option": "dbname", - "default": "amavis", - }, - { - "option": "dbuser", - "default": "amavis", - }, - { - "option": "dbpassword", - "default": make_password, - "customizable": True, - "question": "Please enter amavis db password" - }, - ], - }, - { - "name": "clamav", - "values": [ - { - "option": "enabled", - "default": "true", - }, - { - "option": "user", - "default": "clamav", - }, - ] - }, - { - "name": "dovecot", - "values": [ - { - "option": "enabled", - "default": "true", - }, - { - "option": "config_dir", - "default": "/etc/dovecot", - }, - { - "option": "user", - "default": "vmail", - }, - { - "option": "home_dir", - "default": "/srv/vmail", - }, - { - "option": "mailboxes_owner", - "default": "vmail", - }, - { - "option": "extra_protocols", - "default": "", - }, - { - "option": "postmaster_address", - "default": "postmaster@%(domain)s", - }, - ] - }, - { - "name": "nginx", - "values": [ - { - "option": "enabled", - "default": "true", - }, - { - "option": "config_dir", - "default": "/etc/nginx", - }, - ], - }, - { - "name": "razor", - "values": [ - { - "option": "enabled", - "default": "true", - }, - { - "option": "config_dir", - "default": "/etc/razor", - }, - ] - }, - { - "name": "postfix", - "values": [ - { - "option": "enabled", - "default": "true", - }, - { - "option": "config_dir", - "default": "/etc/postfix", - }, - { - "option": "message_size_limit", - "default": "11534336", - }, - ] - }, - { - "name": "spamassassin", - "values": [ - { - "option": "enabled", - "default": "true", - }, - { - "option": "config_dir", - "default": "/etc/mail/spamassassin", - }, - { - "option": "dbname", - "default": "spamassassin", - }, - { - "option": "dbuser", - "default": "spamassassin", - }, - { - "option": "dbpassword", - "default": make_password, - "customizable": True, - "question": "Please enter spamassassin db password" - }, - ] - }, - { - "name": "uwsgi", - "values": [ - { - "option": "enabled", - "default": "true", - }, - { - "option": "config_dir", - "default": "/etc/uwsgi", - }, - { - "option": "nb_processes", - "default": "2", - }, - ] - }, + { + "name": "general", + "values": [ + { + "option": "hostname", + "default": "mail.%(domain)s", + } + ] + }, + { + "name": "certificate", + "values": [ + { + "option": "generate", + "default": "true", + }, + { + "option": "type", + "default": "self-signed", + "customizable": True, + "question": "Please choose your certificate type", + "values": ["self-signed", "letsencrypt"], + } + ], + }, + { + "name": "letsencrypt", + "if": "certificate.type=letsencrypt", + "values": [ + { + "option": "email", + "default": "admin@example.com", + "question": ( + "Please enter the mail you wish to use for " + "letsencrypt"), + "customizable": True, + "validators": [is_email] + } + ] + }, + { + "name": "database", + "values": [ + { + "option": "engine", + "default": "postgres", + "customizable": True, + "question": "Please choose your database engine", + "values": ["postgres", "mysql"], + }, + { + "option": "host", + "default": "127.0.0.1", + }, + { + "option": "install", + "default": "true", + } + ] + }, + { + "name": "postgres", + "if": "database.engine=postgres", + "values": [ + { + "option": "user", + "default": "postgres", + }, + { + "option": "password", + "default": "", + "customizable": True, + "question": "Please enter postgres password", + }, + ] + }, + { + "name": "mysql", + "if": "database.engine=mysql", + "values": [ + { + "option": "user", + "default": "root", + }, + { + "option": "password", + "default": make_password, + "customizable": True, + "question": "Please enter mysql root password" + }, + { + "option": "charset", + "default": "utf8", + }, + { + "option": "collation", + "default": "utf8_general_ci", + } + ] + }, + { + "name": "modoboa", + "values": [ + { + "option": "user", + "default": "modoboa", + }, + { + "option": "home_dir", + "default": "/srv/modoboa", + }, + { + "option": "venv_path", + "default": "%(home_dir)s/env", + }, + { + "option": "instance_path", + "default": "%(home_dir)s/instance", + }, + { + "option": "timezone", + "default": "Europe/Paris", + }, + { + "option": "dbname", + "default": "modoboa", + }, + { + "option": "dbuser", + "default": "modoboa", + }, + { + "option": "dbpassword", + "default": make_password, + "customizable": True, + "question": "Please enter Modoboa db password", + }, + { + "option": "extensions", + "default": ( + "modoboa-amavis modoboa-pdfcredentials " + "modoboa-postfix-autoreply modoboa-sievefilters " + "modoboa-stats modoboa-webmail modoboa-contacts"), + }, + { + "option": "devmode", + "default": "false", + }, + ] + }, + { + "name": "automx", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "user", + "default": "automx", + }, + { + "option": "config_dir", + "default": "/etc", + }, + { + "option": "home_dir", + "default": "/srv/automx", + }, + { + "option": "venv_path", + "default": "%(home_dir)s/env", + }, + { + "option": "instance_path", + "default": "%(home_dir)s/instance", + }, + ] + }, + { + "name": "amavis", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "user", + "default": "amavis", + }, + { + "option": "max_servers", + "default": "1", + }, + { + "option": "dbname", + "default": "amavis", + }, + { + "option": "dbuser", + "default": "amavis", + }, + { + "option": "dbpassword", + "default": make_password, + "customizable": True, + "question": "Please enter amavis db password" + }, + ], + }, + { + "name": "clamav", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "user", + "default": "clamav", + }, + ] + }, + { + "name": "dovecot", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/dovecot", + }, + { + "option": "user", + "default": "vmail", + }, + { + "option": "home_dir", + "default": "/srv/vmail", + }, + { + "option": "mailboxes_owner", + "default": "vmail", + }, + { + "option": "extra_protocols", + "default": "", + }, + { + "option": "postmaster_address", + "default": "postmaster@%(domain)s", + }, + ] + }, + { + "name": "nginx", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/nginx", + }, + ], + }, + { + "name": "razor", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/razor", + }, + ] + }, + { + "name": "postfix", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/postfix", + }, + { + "option": "message_size_limit", + "default": "11534336", + }, + ] + }, + { + "name": "postwhite", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc", + }, + ] + }, + { + "name": "spamassassin", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/mail/spamassassin", + }, + { + "option": "dbname", + "default": "spamassassin", + }, + { + "option": "dbuser", + "default": "spamassassin", + }, + { + "option": "dbpassword", + "default": make_password, + "customizable": True, + "question": "Please enter spamassassin db password" + }, + ] + }, + { + "name": "uwsgi", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "config_dir", + "default": "/etc/uwsgi", + }, + { + "option": "nb_processes", + "default": "2", + }, + ] + }, ] diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 7fdb89a..0b897fb 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -14,7 +14,7 @@ class Installer(object): appname = None no_daemon = False daemon_name = None - packages = [] + packages = {} with_user = False with_db = False config_files = [] @@ -97,7 +97,7 @@ class Installer(object): def get_packages(self): """Return the list of packages to install.""" - return self.packages[package.backend.FORMAT] + return self.packages.get(package.backend.FORMAT, {}) def install_packages(self): """Install required packages.""" diff --git a/modoboa_installer/scripts/files/postfix/main.cf.tpl b/modoboa_installer/scripts/files/postfix/main.cf.tpl index fc87268..174a654 100644 --- a/modoboa_installer/scripts/files/postfix/main.cf.tpl +++ b/modoboa_installer/scripts/files/postfix/main.cf.tpl @@ -135,6 +135,7 @@ smtpd_recipient_restrictions = # postscreen_access_list = permit_mynetworks + cidr:/etc/postfix/postscreen_spf_whitelist.cidr postscreen_blacklist_action = enforce # Use some DNSBL diff --git a/modoboa_installer/scripts/files/postwhite/crontab.tpl b/modoboa_installer/scripts/files/postwhite/crontab.tpl new file mode 100644 index 0000000..ffc6626 --- /dev/null +++ b/modoboa_installer/scripts/files/postwhite/crontab.tpl @@ -0,0 +1,9 @@ +# +# Postwhite specific cron jobs +# + +# Update Postscreen Whitelists +@daily root /usr/local/bin/postwhite/postwhite > /dev/null 2>&1 + +# Update Yahoo! IPs for Postscreen Whitelists +@weekly root /usr/local/bin/postwhite/scrape_yahoo > /dev/null 2>&1 diff --git a/modoboa_installer/scripts/postfix.py b/modoboa_installer/scripts/postfix.py index 4271ce9..5c95c84 100644 --- a/modoboa_installer/scripts/postfix.py +++ b/modoboa_installer/scripts/postfix.py @@ -10,6 +10,7 @@ from .. import package from .. import utils from . import base +from . import install class Postfix(base.Installer): @@ -92,3 +93,6 @@ class Postfix(base.Installer): aliases_file = "/etc/aliases" if os.path.exists(aliases_file): utils.exec_cmd("postalias {}".format(aliases_file)) + + # Postwhite + install("postwhite", self.config) diff --git a/modoboa_installer/scripts/postwhite.py b/modoboa_installer/scripts/postwhite.py new file mode 100644 index 0000000..239d2d3 --- /dev/null +++ b/modoboa_installer/scripts/postwhite.py @@ -0,0 +1,51 @@ +"""postwhite related functions.""" + +import os +import shutil + +from .. import utils + +from . import base + +POSTWHITE_REPOSITORY = "https://github.com/stevejenkins/postwhite" +SPF_TOOLS_REPOSITORY = "https://github.com/jsarenik/spf-tools" + + +class Postwhite(base.Installer): + """Postwhite installer.""" + + appname = "postwhite" + config_files = [ + "crontab=/etc/cron.d/postwhite", + ] + no_daemon = True + packages = { + "rpm": ["bind-utils"] + } + + def install_from_archive(self, repository, target_dir): + """Install from an archive.""" + url = "{}/archive/master.zip".format(repository) + target = os.path.join(target_dir, os.path.basename(url)) + if os.path.exists(target): + os.unlink(target) + utils.exec_cmd("wget {}".format(url), cwd=target_dir) + app_name = os.path.basename(repository) + archive_dir = os.path.join(target_dir, app_name) + if os.path.exists(archive_dir): + shutil.rmtree(archive_dir) + utils.exec_cmd("unzip master.zip", cwd=target_dir) + utils.exec_cmd( + "mv {name}-master {name}".format(name=app_name), cwd=target_dir) + os.unlink(target) + return archive_dir + + def post_run(self): + """Additionnal tasks.""" + install_dir = "/usr/local/bin" + self.install_from_archive(SPF_TOOLS_REPOSITORY, install_dir) + postw_dir = self.install_from_archive( + POSTWHITE_REPOSITORY, install_dir) + utils.copy_file(os.path.join(postw_dir, "postwhite.conf"), "/etc") + postw_bin = os.path.join(postw_dir, "postwhite") + utils.exec_cmd("{} /etc/postwhite.conf".format(postw_bin)) diff --git a/tests.py b/tests.py index 7b86310..67068bd 100644 --- a/tests.py +++ b/tests.py @@ -87,7 +87,7 @@ class ConfigFileTestCase(unittest.TestCase): self.assertTrue(os.path.exists(self.cfgfile)) self.assertIn( "modoboa automx amavis clamav dovecot nginx razor postfix" - " spamassassin uwsgi", + " postwhite spamassassin uwsgi", out.getvalue() ) From 90e30e5b3e5cb4e6c18a6a4b825337b74b468142 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 8 Dec 2017 14:42:32 +0100 Subject: [PATCH 39/56] Change renewal authenticator to nginx. fix #165 --- modoboa_installer/ssl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modoboa_installer/ssl.py b/modoboa_installer/ssl.py index 87aebee..2a295ec 100644 --- a/modoboa_installer/ssl.py +++ b/modoboa_installer/ssl.py @@ -84,6 +84,9 @@ class LetsEncryptCertificate(CertificateBackend): "--post-hook 'service nginx start && " "service postfix reload && " "service dovecot reload'") + cfg_file = "/etc/letsencrypt/renewal/{}.conf".format(hostname) + pattern = "s/authenticator = standalone/authenticator = nginx/" + utils.exec_cmd("perl -pi -e '{}' {}".format(pattern, cfg_file)) def get_backend(config): From 8a210c872479c13acdbec5865e595f78dd5a46da Mon Sep 17 00:00:00 2001 From: "Tuxis Internet Engineering V.O.F" Date: Thu, 14 Dec 2017 16:43:11 +0100 Subject: [PATCH 40/56] Fix the auth_mechanisms setting in dovecot/10-auth.conf. Closes #175 (#176) --- modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf b/modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf index ce2d259..0db3552 100644 --- a/modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf +++ b/modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf @@ -96,7 +96,7 @@ auth_master_user_separator = * # plain login digest-md5 cram-md5 ntlm rpa apop anonymous gssapi otp skey # gss-spnego # NOTE: See also disable_plaintext_auth setting. -auth_mechanisms = plain +auth_mechanisms = plain login cram-md5 ## ## Password and user databases From b88df74ec23ca53d1d75b83f5fe41cb1c603cfc5 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 22 Dec 2017 10:37:05 +0100 Subject: [PATCH 41/56] Removed cram-md5 auth mechanism. --- modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf b/modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf index 0db3552..b6d3283 100644 --- a/modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf +++ b/modoboa_installer/scripts/files/dovecot/conf.d/10-auth.conf @@ -96,7 +96,7 @@ auth_master_user_separator = * # plain login digest-md5 cram-md5 ntlm rpa apop anonymous gssapi otp skey # gss-spnego # NOTE: See also disable_plaintext_auth setting. -auth_mechanisms = plain login cram-md5 +auth_mechanisms = plain login ## ## Password and user databases From 1d13e8a0e46d47ba2d7b4d88d6dcf623808bd0eb Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Mon, 29 Jan 2018 09:31:12 +0100 Subject: [PATCH 42/56] Updated configuration for 1.10.0 --- modoboa_installer/scripts/files/modoboa/crontab.tpl | 3 +++ modoboa_installer/scripts/files/postfix/main.cf.tpl | 13 ++++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/modoboa_installer/scripts/files/modoboa/crontab.tpl b/modoboa_installer/scripts/files/modoboa/crontab.tpl index f58c489..6cb2a70 100644 --- a/modoboa_installer/scripts/files/modoboa/crontab.tpl +++ b/modoboa_installer/scripts/files/modoboa/crontab.tpl @@ -31,3 +31,6 @@ INSTANCE=%{instance_path} # Public API communication 0 * * * * root $PYTHON $INSTANCE/manage.py communicate_with_public_api + +# Generate DKIM keys (they will belong to the user running this job) +* * * * * root $PYTHON $INSTANCE/manage.py modo manage_dkim_keys diff --git a/modoboa_installer/scripts/files/postfix/main.cf.tpl b/modoboa_installer/scripts/files/postfix/main.cf.tpl index 174a654..093137a 100644 --- a/modoboa_installer/scripts/files/postfix/main.cf.tpl +++ b/modoboa_installer/scripts/files/postfix/main.cf.tpl @@ -28,15 +28,12 @@ proxy_read_maps = proxy:%{db_driver}:/etc/postfix/sql-domain-aliases.cf proxy:%{db_driver}:/etc/postfix/sql-aliases.cf proxy:%{db_driver}:/etc/postfix/sql-relaydomains.cf - proxy:%{db_driver}:/etc/postfix/sql-relaydomains-transport.cf - proxy:%{db_driver}:/etc/postfix/sql-relaydomain-aliases-transport.cf proxy:%{db_driver}:/etc/postfix/sql-autoreplies-transport.cf proxy:%{db_driver}:/etc/postfix/sql-maintain.cf proxy:%{db_driver}:/etc/postfix/sql-relay-recipient-verification.cf - proxy:%{db_driver}:/etc/postfix/sql-sender-login-mailboxes.cf - proxy:%{db_driver}:/etc/postfix/sql-sender-login-aliases.cf - proxy:%{db_driver}:/etc/postfix/sql-sender-login-mailboxes-extra.cf + proxy:%{db_driver}:/etc/postfix/sql-sender-login-map.cf proxy:%{db_driver}:/etc/postfix/sql-spliteddomains-transport.cf + proxy:%{db_driver}:/etc/postfix/sql-transport.cf ## TLS settings # @@ -79,8 +76,8 @@ virtual_alias_maps = relay_domains = proxy:%{db_driver}:/etc/postfix/sql-relaydomains.cf transport_maps = + proxy:%{db_driver}:/etc/postfix/sql-transport.cf proxy:%{db_driver}:/etc/postfix/sql-spliteddomains-transport.cf - proxy:%{db_driver}:/etc/postfix/sql-relaydomains-transport.cf proxy:%{db_driver}:/etc/postfix/sql-autoreplies-transport.cf ## SASL authentication through Dovecot @@ -114,9 +111,7 @@ strict_rfc821_envelopes = yes # List of authorized senders smtpd_sender_login_maps = - proxy:%{db_driver}:/etc/postfix/sql-sender-login-mailboxes.cf - proxy:%{db_driver}:/etc/postfix/sql-sender-login-aliases.cf - proxy:%{db_driver}:/etc/postfix/sql-sender-login-mailboxes-extra.cf + proxy:%{db_driver}:/etc/postfix/sql-sender-login-map.cf # Recipient restriction rules smtpd_recipient_restrictions = From 573255fb320b301180b473d446362c5fa7d33176 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 28 Feb 2018 08:29:42 +0100 Subject: [PATCH 43/56] Removed deprecated option. --- modoboa_installer/scripts/files/modoboa/crontab.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/files/modoboa/crontab.tpl b/modoboa_installer/scripts/files/modoboa/crontab.tpl index 6cb2a70..9967dc6 100644 --- a/modoboa_installer/scripts/files/modoboa/crontab.tpl +++ b/modoboa_installer/scripts/files/modoboa/crontab.tpl @@ -17,7 +17,7 @@ INSTANCE=%{instance_path} %{amavis_enabled}0 0 * * * root $PYTHON $INSTANCE/manage.py qcleanup # Notifications about pending release requests -%{amavis_enabled}0 12 * * * root $PYTHON $INSTANCE/manage.py amnotify --baseurl='http://%{hostname}' +%{amavis_enabled}0 12 * * * root $PYTHON $INSTANCE/manage.py amnotify # Logs parsing */5 * * * * root $PYTHON $INSTANCE/manage.py logparser &> /dev/null From 18022f6941ee1dc14042b47d0e42a3e40d76847f Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 9 Mar 2018 13:19:21 +0100 Subject: [PATCH 44/56] Use CA certs to identify TLS peers. see https://github.com/modoboa/modoboa/issues/1428 --- modoboa_installer/scripts/files/postfix/main.cf.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modoboa_installer/scripts/files/postfix/main.cf.tpl b/modoboa_installer/scripts/files/postfix/main.cf.tpl index 093137a..ae57d3c 100644 --- a/modoboa_installer/scripts/files/postfix/main.cf.tpl +++ b/modoboa_installer/scripts/files/postfix/main.cf.tpl @@ -39,6 +39,7 @@ proxy_read_maps = # smtpd_use_tls = yes smtpd_tls_auth_only = no +smtpd_tls_CApath = /etc/ssl/certs smtpd_tls_key_file = %tls_key_file smtpd_tls_cert_file = %tls_cert_file smtpd_tls_dh1024_param_file = ${config_directory}/dh2048.pem @@ -58,6 +59,7 @@ smtpd_tls_exclude_ciphers = aNULL, MD5 , DES, ADH, RC4, PSD, SRP, 3DES, eNULL smtpd_tls_eecdh_grade = strong # Use TLS if this is supported by the remote SMTP server, otherwise use plaintext. +smtp_tls_CApath = /etc/ssl/certs smtp_tls_security_level = may smtp_tls_loglevel = 1 smtp_tls_exclude_ciphers = EXPORT, LOW From 6ea1114cd8e3b71f9904cc141524a78cea52c528 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 9 Mar 2018 15:31:58 +0100 Subject: [PATCH 45/56] Updated database connectors. fix #185 --- modoboa_installer/scripts/automx.py | 4 ++-- modoboa_installer/scripts/modoboa.py | 12 ++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/modoboa_installer/scripts/automx.py b/modoboa_installer/scripts/automx.py index 89e67db..2c9f5b0 100644 --- a/modoboa_installer/scripts/automx.py +++ b/modoboa_installer/scripts/automx.py @@ -59,9 +59,9 @@ class Automx(base.Installer): "python-dateutil", "configparser" ] if self.dbengine == "postgres": - packages.append("psycopg2") + packages.append("psycopg2-binary") else: - packages.append("MYSQL-Python") + packages.append("mysqlclient") python.install_packages(packages, self.venv_path, sudo_user=self.user) target = "{}/master.zip".format(self.home_dir) if os.path.exists(target): diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index fa7046c..e7d933f 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -82,9 +82,9 @@ class Modoboa(base.Installer): else: packages.append(extension) if self.dbengine == "postgres": - packages.append("psycopg2") + packages.append("psycopg2-binary") else: - packages.append("MYSQL-Python") + packages.append("mysqlclient") if sys.version_info.major == 2 and sys.version_info.micro < 9: # Add extra packages to fix the SNI issue packages += ["pyOpenSSL"] @@ -192,14 +192,6 @@ class Modoboa(base.Installer): "handle_mailboxes": True, "account_auto_removal": True }, - # FIXME: since we rewrite all parameters, the secret key - # previously created will disappear. As a quick fix, we - # recreate a new one here but it will mess up opened - # sessions if the installer is used to upgrade an existing - # database... - "core": { - "secret_key": utils.random_key() - }, "modoboa_amavis": { "am_pdp_mode": "inet", }, From b68de9e13976389730c370328de118019f9af1e6 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 30 Mar 2018 14:26:23 +0200 Subject: [PATCH 46/56] Added missing dependency. fix #191 --- modoboa_installer/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modoboa_installer/database.py b/modoboa_installer/database.py index 9b4edd4..2a9db0e 100644 --- a/modoboa_installer/database.py +++ b/modoboa_installer/database.py @@ -125,7 +125,7 @@ class MySQL(Database): """MySQL backend.""" packages = { - "deb": ["mariadb-server"], + "deb": ["mariadb-server", "libmysqlclient-dev"], "rpm": ["mariadb", "mariadb-devel", "mariadb-server"], } service = "mariadb" From d8130392706dc1c2ff1fd938629ae3d7db839942 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sat, 31 Mar 2018 14:12:53 +0200 Subject: [PATCH 47/56] Added Radicale setup. (#194) * Added Radicale setup. see #193 * Fixed setup on CentOS. --- modoboa_installer/config_dict_template.py | 33 +++- modoboa_installer/python.py | 31 +++- modoboa_installer/scripts/dovecot.py | 3 +- .../files/dovecot/conf.d/10-master.conf.tpl | 7 + .../scripts/files/radicale/config.tpl | 161 ++++++++++++++++++ .../scripts/files/radicale/supervisor.tpl | 8 + modoboa_installer/scripts/modoboa.py | 9 + modoboa_installer/scripts/nginx.py | 9 + modoboa_installer/scripts/radicale.py | 79 +++++++++ run.py | 1 + 10 files changed, 334 insertions(+), 7 deletions(-) create mode 100644 modoboa_installer/scripts/files/radicale/config.tpl create mode 100644 modoboa_installer/scripts/files/radicale/supervisor.tpl create mode 100644 modoboa_installer/scripts/radicale.py diff --git a/modoboa_installer/config_dict_template.py b/modoboa_installer/config_dict_template.py index 6f39dc9..7e15a74 100644 --- a/modoboa_installer/config_dict_template.py +++ b/modoboa_installer/config_dict_template.py @@ -158,7 +158,9 @@ ConfigDictTemplate = [ "default": ( "modoboa-amavis modoboa-pdfcredentials " "modoboa-postfix-autoreply modoboa-sievefilters " - "modoboa-stats modoboa-webmail modoboa-contacts"), + "modoboa-stats modoboa-webmail modoboa-contacts " + "modoboa-radicale" + ), }, { "option": "devmode", @@ -270,6 +272,10 @@ ConfigDictTemplate = [ "option": "postmaster_address", "default": "postmaster@%(domain)s", }, + { + "option": "radicale_auth_socket_path", + "default": "/var/run/dovecot/auth-radicale" + }, ] }, { @@ -372,4 +378,29 @@ ConfigDictTemplate = [ }, ] }, + { + "name": "radicale", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "user", + "default": "radicale", + }, + { + "option": "config_dir", + "default": "/etc/radicale", + }, + { + "option": "home_dir", + "default": "/srv/radicale", + }, + { + "option": "venv_path", + "default": "%(home_dir)s/env", + } + ] + }, ] diff --git a/modoboa_installer/python.py b/modoboa_installer/python.py index 87edd93..3c07f19 100644 --- a/modoboa_installer/python.py +++ b/modoboa_installer/python.py @@ -36,14 +36,35 @@ def install_packages(names, venv=None, upgrade=False, **kwargs): utils.exec_cmd(cmd, **kwargs) -def setup_virtualenv(path, sudo_user=None): +def install_package_from_repository(name, url, vcs="git", venv=None, **kwargs): + """Install a Python package from its repository.""" + if vcs == "git": + package.backend.install("git") + cmd = "{} install -e {}+{}#egg={}".format( + get_pip_path(venv), vcs, url, name) + utils.exec_cmd(cmd, **kwargs) + + +def setup_virtualenv(path, sudo_user=None, python_version=2): """Install a virtualenv if needed.""" if os.path.exists(path): return - packages = ["python-virtualenv"] - if utils.dist_name() == "debian": - packages.append("virtualenv") + if python_version == 2: + python_binary = "python" + packages = ["python-virtualenv"] + if utils.dist_name() == "debian": + packages.append("virtualenv") + else: + if utils.dist_name().startswith("centos"): + python_binary = "python36" + packages = ["python36"] + else: + python_binary = "python3" + packages = ["python3-venv"] package.backend.install_many(packages) with utils.settings(sudo_user=sudo_user): - utils.exec_cmd("virtualenv {}".format(path)) + if python_version == 2: + utils.exec_cmd("virtualenv {}".format(path)) + else: + utils.exec_cmd("{} -m venv {}".format(python_binary, path)) install_package("pip", venv=path, upgrade=True) diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 04ce6b4..093fbcf 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -77,7 +77,8 @@ class Dovecot(base.Installer): "modoboa_dbuser": self.config.get("modoboa", "dbuser"), "modoboa_dbpassword": self.config.get("modoboa", "dbpassword"), "protocols": protocols, - "ssl_protocols": ssl_protocols + "ssl_protocols": ssl_protocols, + "radicale_user": self.config.get("radicale", "user") }) return context diff --git a/modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf.tpl b/modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf.tpl index 740f169..6c25b8d 100644 --- a/modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf.tpl +++ b/modoboa_installer/scripts/files/dovecot/conf.d/10-master.conf.tpl @@ -116,6 +116,13 @@ service auth { group = postfix } + # Radicale auth + %{radicale_enabled}unix_listener %{radicale_auth_socket_path} { + %{radicale_enabled} mode = 0666 + %{radicale_enabled} user = %{radicale_user} + %{radicale_enabled} group = %{radicale_user} + %{radicale_enabled}} + # Auth process is run as this user. #user = $default_internal_user } diff --git a/modoboa_installer/scripts/files/radicale/config.tpl b/modoboa_installer/scripts/files/radicale/config.tpl new file mode 100644 index 0000000..a4285a4 --- /dev/null +++ b/modoboa_installer/scripts/files/radicale/config.tpl @@ -0,0 +1,161 @@ +# -*- mode: conf -*- +# vim:ft=cfg + +# Config file for Radicale - A simple calendar server +# +# Place it into /etc/radicale/config (global) +# or ~/.config/radicale/config (user) +# +# The current values are the default ones + + +[server] + +# CalDAV server hostnames separated by a comma +# IPv4 syntax: address:port +# IPv6 syntax: [address]:port +# For example: 0.0.0.0:9999, [::]:9999 +#hosts = 127.0.0.1:5232 + +# Daemon flag +#daemon = False + +# File storing the PID in daemon mode +#pid = + +# Max parallel connections +#max_connections = 20 + +# Max size of request body (bytes) +#max_content_length = 10000000 + +# Socket timeout (seconds) +#timeout = 10 + +# SSL flag, enable HTTPS protocol +#ssl = False + +# SSL certificate path +#certificate = /etc/ssl/radicale.cert.pem + +# SSL private key +#key = /etc/ssl/radicale.key.pem + +# CA certificate for validating clients. This can be used to secure +# TCP traffic between Radicale and a reverse proxy +#certificate_authority = + +# SSL Protocol used. See python's ssl module for available values +#protocol = PROTOCOL_TLSv1_2 + +# Available ciphers. See python's ssl module for available ciphers +#ciphers = + +# Reverse DNS to resolve client address in logs +#dns_lookup = True + +# Message displayed in the client when a password is needed +#realm = Radicale - Password Required + + +[encoding] + +# Encoding for responding requests +#request = utf-8 + +# Encoding for storing local collections +#stock = utf-8 + + +[auth] + +# Authentication method +# Value: none | htpasswd | remote_user | http_x_remote_user +type = radicale_dovecot_auth + +# Htpasswd filename +# htpasswd_filename = users + +# Htpasswd encryption method +# Value: plain | sha1 | ssha | crypt | bcrypt | md5 +# Only bcrypt can be considered secure. +# bcrypt and md5 require the passlib library to be installed. +# htpasswd_encryption = plain + +# Incorrect authentication delay (seconds) +#delay = 1 + +auth_socket = %{radicale_auth_socket_path} + + +[rights] + +# Rights backend +# Value: none | authenticated | owner_only | owner_write | from_file +type = from_file + +# File for rights management from_file +file = %{config_dir}/rights + + +[storage] + +# Storage backend +# Value: multifilesystem +type = radicale_storage_by_index +radicale_storage_by_index_fields = dtstart, dtend, uid, summary + +# Folder for storing local collections, created if not present +filesystem_folder = %{home_dir}/collections + +# Lock the storage. Never start multiple instances of Radicale or edit the +# storage externally while Radicale is running if disabled. +#filesystem_locking = True + +# Sync all changes to disk during requests. (This can impair performance.) +# Disabling it increases the risk of data loss, when the system crashes or +# power fails! +#filesystem_fsync = True + +# Delete sync token that are older (seconds) +#max_sync_token_age = 2592000 + +# Close the lock file when no more clients are waiting. +# This option is not very useful in general, but on Windows files that are +# opened cannot be deleted. +#filesystem_close_lock_file = False + +# Command that is run after changes to storage +# Example: ([ -d .git ] || git init) && git add -A && (git diff --cached --quiet || git commit -m "Changes by "%%(user)s) +#hook = + + +[web] + +# Web interface backend +# Value: none | internal +type = none + + +[logging] + +# Logging configuration file +# If no config is given, simple information is printed on the standard output +# For more information about the syntax of the configuration file, see: +# http://docs.python.org/library/logging.config.html +#config = /etc/radicale/logging + +# Set the default logging level to debug +debug = False + +# Store all environment variables (including those set in the shell) +#full_environment = False + +# Don't include passwords in logs +#mask_passwords = True + + +[headers] + +# Additional HTTP headers +#Access-Control-Allow-Origin = * diff --git a/modoboa_installer/scripts/files/radicale/supervisor.tpl b/modoboa_installer/scripts/files/radicale/supervisor.tpl new file mode 100644 index 0000000..644f337 --- /dev/null +++ b/modoboa_installer/scripts/files/radicale/supervisor.tpl @@ -0,0 +1,8 @@ +[program:radicale] +autostart=true +autorestart=true +command=%{venv_path}/bin/radicale -C %{config_dir}/config +directory=%{home_dir} +redirect_stderr=true +user=%{user} +numprocs=1 diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index e7d933f..97a7a74 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -50,6 +50,9 @@ class Modoboa(base.Installer): self.amavis_enabled = True else: self.extensions.remove("modoboa-amavis") + if "modoboa-radicale" in self.extensions: + if not self.config.getboolean("radicale", "enabled"): + self.extensions.remove("modoboa-radicale") def is_extension_ok_for_version(self, extension, version): """Check if extension can be installed with this modo version.""" @@ -200,6 +203,12 @@ class Modoboa(base.Installer): }, "modoboa_pdfcredentials": { "storage_dir": pdf_storage_dir + }, + "modoboa_radicale": { + "server_location": "https://{}/radicale/".format( + self.config.get("general", "hostname")), + "rights_file_path": "{}/rights".format( + self.config.get("radicale", "config_dir")) } } for path in ["/var/log/maillog", "/var/log/mail.log"]: diff --git a/modoboa_installer/scripts/nginx.py b/modoboa_installer/scripts/nginx.py index def4d78..bbff33a 100644 --- a/modoboa_installer/scripts/nginx.py +++ b/modoboa_installer/scripts/nginx.py @@ -71,6 +71,15 @@ class Nginx(base.Installer): include uwsgi_params; uwsgi_pass automx; } +""" + if self.config.get("radicale", "enabled"): + extra_modoboa_config += """ + location /radicale/ { + proxy_pass http://localhost:5232/; # The / is important! + proxy_set_header X-Script-Name /radicale; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass_header Authorization; + } """ self._setup_config( "modoboa", extra_config=extra_modoboa_config) diff --git a/modoboa_installer/scripts/radicale.py b/modoboa_installer/scripts/radicale.py new file mode 100644 index 0000000..5b4fc75 --- /dev/null +++ b/modoboa_installer/scripts/radicale.py @@ -0,0 +1,79 @@ +"""Radicale related tasks.""" + +import os +import stat + +from .. import package +from .. import python +from .. import utils + +from . import base + + +class Radicale(base.Installer): + """Radicale installation.""" + + appname = "radicale" + config_files = ["config"] + no_daemon = True + packages = { + "deb": ["supervisor"], + "rpm": ["supervisor"] + } + with_user = True + + def __init__(self, config): + """Get configuration.""" + super(Radicale, self).__init__(config) + self.venv_path = config.get("radicale", "venv_path") + + def _setup_venv(self): + """Prepare a dedicated virtualenv.""" + python.setup_virtualenv( + self.venv_path, sudo_user=self.user, python_version=3) + packages = ["Radicale", "radicale-dovecot-auth", "pytz"] + python.install_packages(packages, self.venv_path, sudo_user=self.user) + python.install_package_from_repository( + "radicale-storage-by-index", + "https://github.com/tonioo/RadicaleStorageByIndex", + venv=self.venv_path, sudo_user=self.user) + + def get_template_context(self): + """Additional variables.""" + context = super(Radicale, self).get_template_context() + radicale_auth_socket_path = self.config.get( + "dovecot", "radicale_auth_socket_path") + context.update({ + "radicale_auth_socket_path": radicale_auth_socket_path + }) + return context + + def get_config_files(self): + """Return appropriate path.""" + config_files = super(Radicale, self).get_config_files() + if package.backend.FORMAT == "deb": + path = "supervisor=/etc/supervisor/conf.d/radicale.conf" + else: + path = "supervisor=/etc/supervisord.d/radicale.ini" + config_files.append(path) + return config_files + + def install_config_files(self): + """Make sure config directory exists.""" + if not os.path.exists(self.config_dir): + utils.mkdir( + self.config_dir, + stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH, + 0, 0 + ) + super(Radicale, self).install_config_files() + + def post_run(self): + """Additional tasks.""" + self._setup_venv() + daemon_name = ( + "supervisor" if package.backend.FORMAT == "deb" else "supervisord" + ) + utils.exec_cmd("service {} stop".format(daemon_name)) + utils.exec_cmd("service {} start".format(daemon_name)) diff --git a/run.py b/run.py index 80c586f..d796bba 100755 --- a/run.py +++ b/run.py @@ -93,6 +93,7 @@ def main(input_args): scripts.install("amavis", config) scripts.install("modoboa", config) scripts.install("automx", config) + scripts.install("radicale", config) scripts.install("uwsgi", config) scripts.install("nginx", config) scripts.install("postfix", config) From c4db97ea7abdb27a2ba4787d84954fa8c181c3a0 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Sat, 31 Mar 2018 14:27:37 +0200 Subject: [PATCH 48/56] Updated README. --- README.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.rst b/README.rst index f24c596..bed8d5f 100644 --- a/README.rst +++ b/README.rst @@ -65,6 +65,26 @@ a previous one using the ``--version`` option:: If you want more information about the installation process, add the ``--debug`` option to your command line. +Change the generated hostname +----------------------------- + +By default, the installer will setup your email server using the +following hostname: ``mail.``. If you want a different +value, generate the configuration file like this:: + + $ ./run.py --stop-after-configfile-check + +Then edit ``installer.cfg`` and look for the following section:: + + [general] + hostname = mail.%(domain)s + +Replace ``mail`` by the value you want to use and save your +modifications. + +Finally, run the installer without the +``--stop-after-configfile-check`` option. + Let's Encrypt certificate ------------------------- From 704d73cb4d68824bac0b087aed5171b43f97e6e7 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Mon, 2 Apr 2018 16:23:47 +0200 Subject: [PATCH 49/56] Fixed dovecot warning. --- modoboa_installer/scripts/dovecot.py | 5 ++++- modoboa_installer/scripts/files/radicale/config.tpl | 2 +- modoboa_installer/scripts/radicale.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 093fbcf..3830216 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -1,6 +1,7 @@ """Dovecot related tools.""" import glob +import os import pwd from .. import database @@ -78,7 +79,9 @@ class Dovecot(base.Installer): "modoboa_dbpassword": self.config.get("modoboa", "dbpassword"), "protocols": protocols, "ssl_protocols": ssl_protocols, - "radicale_user": self.config.get("radicale", "user") + "radicale_user": self.config.get("radicale", "user"), + "radicale_auth_socket_path": os.path.basename( + self.config.get("dovecot", "radicale_auth_socket_path")) }) return context diff --git a/modoboa_installer/scripts/files/radicale/config.tpl b/modoboa_installer/scripts/files/radicale/config.tpl index a4285a4..50e7acf 100644 --- a/modoboa_installer/scripts/files/radicale/config.tpl +++ b/modoboa_installer/scripts/files/radicale/config.tpl @@ -85,7 +85,7 @@ type = radicale_dovecot_auth # Incorrect authentication delay (seconds) #delay = 1 -auth_socket = %{radicale_auth_socket_path} +auth_socket = %{auth_socket_path} [rights] diff --git a/modoboa_installer/scripts/radicale.py b/modoboa_installer/scripts/radicale.py index 5b4fc75..a65f1a7 100644 --- a/modoboa_installer/scripts/radicale.py +++ b/modoboa_installer/scripts/radicale.py @@ -44,7 +44,7 @@ class Radicale(base.Installer): radicale_auth_socket_path = self.config.get( "dovecot", "radicale_auth_socket_path") context.update({ - "radicale_auth_socket_path": radicale_auth_socket_path + "auth_socket_path": radicale_auth_socket_path }) return context From 8a650e699843889c607319161357030886d838de Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Mon, 2 Apr 2018 16:25:58 +0200 Subject: [PATCH 50/56] OpenDKIM setup. (#196) * OpenDKIM setup. see #173 * Fixed unit tests. * Fixed mysql syntax. --- modoboa_installer/config_dict_template.py | 36 ++++++++ modoboa_installer/database.py | 31 +++++-- modoboa_installer/scripts/base.py | 2 + .../scripts/files/modoboa/crontab.tpl | 2 +- .../files/opendkim/dkim_view_mysql.sql | 5 ++ .../files/opendkim/dkim_view_postgres.sql | 5 ++ .../scripts/files/opendkim/opendkim.conf.tpl | 89 +++++++++++++++++++ .../scripts/files/opendkim/opendkim.hosts.tpl | 3 + .../scripts/files/postfix/main.cf.tpl | 6 ++ modoboa_installer/scripts/modoboa.py | 7 +- modoboa_installer/scripts/opendkim.py | 78 ++++++++++++++++ modoboa_installer/scripts/postfix.py | 2 + run.py | 1 + tests.py | 4 +- 14 files changed, 259 insertions(+), 12 deletions(-) create mode 100644 modoboa_installer/scripts/files/opendkim/dkim_view_mysql.sql create mode 100644 modoboa_installer/scripts/files/opendkim/dkim_view_postgres.sql create mode 100644 modoboa_installer/scripts/files/opendkim/opendkim.conf.tpl create mode 100644 modoboa_installer/scripts/files/opendkim/opendkim.hosts.tpl create mode 100644 modoboa_installer/scripts/opendkim.py diff --git a/modoboa_installer/config_dict_template.py b/modoboa_installer/config_dict_template.py index 7e15a74..f300364 100644 --- a/modoboa_installer/config_dict_template.py +++ b/modoboa_installer/config_dict_template.py @@ -403,4 +403,40 @@ ConfigDictTemplate = [ } ] }, + { + "name": "opendkim", + "values": [ + { + "option": "enabled", + "default": "true", + }, + { + "option": "user", + "default": "opendkim", + }, + { + "option": "config_dir", + "default": "/etc", + }, + { + "option": "port", + "default": "12345" + }, + { + "option": "keys_storage_dir", + "default": "/var/lib/dkim" + }, + { + "option": "dbuser", + "default": "opendkim", + }, + { + "option": "dbpassword", + "default": make_password, + "customizable": True, + "question": "Please enter OpenDKIM db password" + }, + + ] + }, ] diff --git a/modoboa_installer/database.py b/modoboa_installer/database.py index 2a9db0e..e1a5ed3 100644 --- a/modoboa_installer/database.py +++ b/modoboa_installer/database.py @@ -60,9 +60,11 @@ class PostgreSQL(Database): def _exec_query(self, query, dbname=None, dbuser=None, dbpassword=None): """Exec a postgresql query.""" cmd = "psql" - if dbname and dbuser: - self._setup_pgpass(dbname, dbuser, dbpassword) - cmd += " -h {} -d {} -U {} -w".format(self.dbhost, dbname, dbuser) + if dbname: + cmd += " -d {}".format(dbname) + if dbuser: + self._setup_pgpass(dbname, dbuser, dbpassword) + cmd += " -h {} -U {} -w".format(self.dbhost, dbuser) query = query.replace("'", "'\"'\"'") cmd = "{} -c '{}' ".format(cmd, query) utils.exec_cmd(cmd, sudo_user=self.dbuser) @@ -94,6 +96,12 @@ class PostgreSQL(Database): query = "GRANT ALL ON DATABASE {} TO {}".format(dbname, user) self._exec_query(query) + def grant_right_on_table(self, dbname, table, user, right): + """Grant specific right to user on table.""" + query = "GRANT {} ON {} TO {}".format( + right.upper(), table, user) + self._exec_query(query, dbname=dbname) + def _setup_pgpass(self, dbname, dbuser, dbpasswd): """Setup .pgpass file.""" if self._pgpass_done: @@ -114,10 +122,9 @@ class PostgreSQL(Database): def load_sql_file(self, dbname, dbuser, dbpassword, path): """Load SQL file.""" self._setup_pgpass(dbname, dbuser, dbpassword) - utils.exec_cmd( - "psql -h {} -d {} -U {} -w < {}".format( - self.dbhost, dbname, dbuser, path), - sudo_user=self.dbuser) + cmd = "psql -h {} -d {} -U {} -w < {}".format( + self.dbhost, dbname, dbuser, path) + utils.exec_cmd(cmd, sudo_user=self.dbuser) class MySQL(Database): @@ -125,7 +132,7 @@ class MySQL(Database): """MySQL backend.""" packages = { - "deb": ["mariadb-server", "libmysqlclient-dev"], + "deb": ["mariadb-server"], "rpm": ["mariadb", "mariadb-devel", "mariadb-server"], } service = "mariadb" @@ -140,6 +147,8 @@ class MySQL(Database): if name == "debian": mysql_name = "mysql" if version.startswith("8") else "mariadb" self.packages["deb"].append("lib{}client-dev".format(mysql_name)) + elif name == "ubuntu": + self.packages["deb"].append("libmysqlclient-dev") super(MySQL, self).install_package() if name == "debian" and version.startswith("8"): package.backend.preconfigure( @@ -200,6 +209,12 @@ class MySQL(Database): "GRANT ALL PRIVILEGES ON {}.* to '{}'@'localhost'" .format(dbname, user)) + def grant_right_on_table(self, dbname, table, user, right): + """Grant specific right to user on table.""" + query = "GRANT {} ON {}.{} TO '{}'@'%'".format( + right.upper(), dbname, table, user) + self._exec_query(query) + def load_sql_file(self, dbname, dbuser, dbpassword, path): """Load SQL file.""" utils.exec_cmd( diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 0b897fb..a0f084e 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -22,6 +22,8 @@ class Installer(object): def __init__(self, config): """Get configuration.""" self.config = config + if self.config.has_section(self.appname): + self.app_config = dict(self.config.items(self.appname)) self.dbengine = self.config.get("database", "engine") # Used to install system packages self.db_driver = ( diff --git a/modoboa_installer/scripts/files/modoboa/crontab.tpl b/modoboa_installer/scripts/files/modoboa/crontab.tpl index 9967dc6..45fcf23 100644 --- a/modoboa_installer/scripts/files/modoboa/crontab.tpl +++ b/modoboa_installer/scripts/files/modoboa/crontab.tpl @@ -33,4 +33,4 @@ INSTANCE=%{instance_path} 0 * * * * root $PYTHON $INSTANCE/manage.py communicate_with_public_api # Generate DKIM keys (they will belong to the user running this job) -* * * * * root $PYTHON $INSTANCE/manage.py modo manage_dkim_keys +%{opendkim_enabled}* * * * * %{opendkim_user} $PYTHON $INSTANCE/manage.py modo manage_dkim_keys diff --git a/modoboa_installer/scripts/files/opendkim/dkim_view_mysql.sql b/modoboa_installer/scripts/files/opendkim/dkim_view_mysql.sql new file mode 100644 index 0000000..7f1ed25 --- /dev/null +++ b/modoboa_installer/scripts/files/opendkim/dkim_view_mysql.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE VIEW dkim AS ( + SELECT id, name as domain_name, dkim_private_key_path AS private_key_path, + dkim_key_selector AS selector + FROM admin_domain WHERE enable_dkim=1 +); diff --git a/modoboa_installer/scripts/files/opendkim/dkim_view_postgres.sql b/modoboa_installer/scripts/files/opendkim/dkim_view_postgres.sql new file mode 100644 index 0000000..f3e7d41 --- /dev/null +++ b/modoboa_installer/scripts/files/opendkim/dkim_view_postgres.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE VIEW dkim AS ( + SELECT id, name as domain_name, dkim_private_key_path AS private_key_path, + dkim_key_selector AS selector + FROM admin_domain WHERE enable_dkim +); diff --git a/modoboa_installer/scripts/files/opendkim/opendkim.conf.tpl b/modoboa_installer/scripts/files/opendkim/opendkim.conf.tpl new file mode 100644 index 0000000..890d25c --- /dev/null +++ b/modoboa_installer/scripts/files/opendkim/opendkim.conf.tpl @@ -0,0 +1,89 @@ +# This is a basic configuration that can easily be adapted to suit a standard +# installation. For more advanced options, see opendkim.conf(5) and/or +# /usr/share/doc/opendkim/examples/opendkim.conf.sample. + +# Log to syslog +Syslog yes +LogWhy Yes +SyslogSuccess Yes +# Required to use local socket with MTAs that access the socket as a non- +# privileged user (e.g. Postfix) +UMask 007 + +# Sign for example.com with key in /etc/dkimkeys/dkim.key using +# selector '2007' (e.g. 2007._domainkey.example.com) +#Domain example.com +#KeyFile /etc/dkimkeys/dkim.key +#Selector 2007 + +KeyTable dsn:%{db_driver}://%{db_user}:%{db_password}@%{dbhost}/%{db_name}/table=dkim?keycol=id?datacol=domain_name,selector,private_key_path +SigningTable dsn:%db_driver://%{db_user}:%{db_password}@%{dbhost}/%{db_name}/table=dkim?keycol=domain_name?datacol=id + +# Commonly-used options; the commented-out versions show the defaults. +#Canonicalization simple +#Mode sv +SubDomains yes +Canonicalization relaxed/relaxed + +# Socket smtp://localhost +# +# ## Socket socketspec +# ## +# ## Names the socket where this filter should listen for milter connections +# ## from the MTA. Required. Should be in one of these forms: +# ## +# ## inet:port@address to listen on a specific interface +# ## inet:port to listen on all interfaces +# ## local:/path/to/socket to listen on a UNIX domain socket +# +Socket inet:%{port}@localhost +#Socket local:/var/run/opendkim/opendkim.sock + +## PidFile filename +### default (none) +### +### Name of the file where the filter should write its pid before beginning +### normal operations. +# +PidFile /var/run/opendkim/opendkim.pid + + +# Always oversign From (sign using actual From and a null From to prevent +# malicious signatures header fields (From and/or others) between the signer +# and the verifier. From is oversigned by default in the Debian pacakge +# because it is often the identity key used by reputation systems and thus +# somewhat security sensitive. +OversignHeaders From + +## ResolverConfiguration filename +## default (none) +## +## Specifies a configuration file to be passed to the Unbound library that +## performs DNS queries applying the DNSSEC protocol. See the Unbound +## documentation at http://unbound.net for the expected content of this file. +## The results of using this and the TrustAnchorFile setting at the same +## time are undefined. +## In Debian, /etc/unbound/unbound.conf is shipped as part of the Suggested +## unbound package + +# ResolverConfiguration /etc/unbound/unbound.conf + +## TrustAnchorFile filename +## default (none) +## +## Specifies a file from which trust anchor data should be read when doing +## DNS queries and applying the DNSSEC protocol. See the Unbound documentation +## at http://unbound.net for the expected format of this file. + +# TrustAnchorFile /usr/share/dns/root.key + +## Userid userid +### default (none) +### +### Change to user "userid" before starting normal operation? May include +### a group ID as well, separated from the userid by a colon. +# +UserID %{user} + +ExternalIgnoreList /etc/opendkim.hosts +InternalHosts /etc/opendkim.hosts diff --git a/modoboa_installer/scripts/files/opendkim/opendkim.hosts.tpl b/modoboa_installer/scripts/files/opendkim/opendkim.hosts.tpl new file mode 100644 index 0000000..46a0ab8 --- /dev/null +++ b/modoboa_installer/scripts/files/opendkim/opendkim.hosts.tpl @@ -0,0 +1,3 @@ +127.0.0.1 +::1 +localhost diff --git a/modoboa_installer/scripts/files/postfix/main.cf.tpl b/modoboa_installer/scripts/files/postfix/main.cf.tpl index ae57d3c..7fc38c5 100644 --- a/modoboa_installer/scripts/files/postfix/main.cf.tpl +++ b/modoboa_installer/scripts/files/postfix/main.cf.tpl @@ -111,6 +111,12 @@ strict_rfc821_envelopes = yes %{dovecot_enabled} $lmtp_sasl_auth_cache_name %{dovecot_enabled} $address_verify_map +# OpenDKIM setup +%{opendkim_enabled}smtpd_milters = inet:127.0.0.1:%{opendkim_port} +%{opendkim_enabled}non_smtpd_milters = inet:127.0.0.1:%{opendkim_port} +%{opendkim_enabled}milter_default_action = accept +%{opendkim_enabled}milter_content_timeout = 30s + # List of authorized senders smtpd_sender_login_maps = proxy:%{db_driver}:/etc/postfix/sql-sender-login-map.cf diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 97a7a74..f27db93 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -177,7 +177,9 @@ class Modoboa(base.Installer): ), "dovecot_mailboxes_owner": ( self.config.get("dovecot", "mailboxes_owner")), - "radicale_enabled": "" if "modoboa-radicale" in extensions else "#" + "radicale_enabled": ( + "" if "modoboa-radicale" in extensions else "#"), + "opendkim_user": self.config.get("opendkim", "user"), }) return context @@ -214,6 +216,9 @@ class Modoboa(base.Installer): for path in ["/var/log/maillog", "/var/log/mail.log"]: if os.path.exists(path): settings["modoboa_stats"]["logfile"] = path + if self.config.getboolean("opendkim", "enabled"): + settings["admin"]["dkim_keys_storage_dir"] = ( + self.config.get("opendkim", "keys_storage_dir")) settings = json.dumps(settings) query = ( "UPDATE core_localconfig SET _parameters='{}'" diff --git a/modoboa_installer/scripts/opendkim.py b/modoboa_installer/scripts/opendkim.py new file mode 100644 index 0000000..6d904af --- /dev/null +++ b/modoboa_installer/scripts/opendkim.py @@ -0,0 +1,78 @@ +"""OpenDKIM related tools.""" + +import os +import pwd +import stat + +from .. import database +from .. import package +from .. import utils + +from . import base + + +class Opendkim(base.Installer): + """OpenDKIM installer.""" + + appname = "opendkim" + packages = { + "deb": ["opendkim"], + "rpm": ["opendkim"] + } + config_files = ["opendkim.conf", "opendkim.hosts"] + + def get_packages(self): + """Additional packages.""" + packages = super(Opendkim, self).get_packages() + if package.backend.FORMAT == "deb": + packages += ["libopendbx1-{}".format(self.db_driver)] + else: + dbengine = "postgresql" if self.dbengine == "postgres" else "mysql" + packages += ["opendbx-{}".format(dbengine)] + return packages + + def install_config_files(self): + """Make sure config directory exists.""" + user = self.config.get("opendkim", "user") + pw = pwd.getpwnam(user) + targets = [ + [self.app_config["keys_storage_dir"], pw[2], pw[3]] + ] + for target in targets: + if not os.path.exists(target[0]): + utils.mkdir( + target[0], + stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH, + target[1], target[2] + ) + super(Opendkim, self).install_config_files() + + def get_template_context(self): + """Additional variables.""" + context = super(Opendkim, self).get_template_context() + context.update({ + "db_driver": self.db_driver, + "db_name": self.config.get("modoboa", "dbname"), + "db_user": self.app_config["dbuser"], + "db_password": self.app_config["dbpassword"], + "port": self.app_config["port"], + "user": self.app_config["user"] + }) + return context + + def setup_database(self): + """Setup database.""" + self.backend = database.get_backend(self.config) + self.backend.create_user( + self.app_config["dbuser"], self.app_config["dbpassword"] + ) + dbname = self.config.get("modoboa", "dbname") + dbuser = self.config.get("modoboa", "dbuser") + dbpassword = self.config.get("modoboa", "dbpassword") + self.backend.load_sql_file( + dbname, dbuser, dbpassword, + self.get_file_path("dkim_view_{}.sql".format(self.dbengine)) + ) + self.backend.grant_right_on_table( + dbname, "dkim", self.app_config["dbuser"], "SELECT") diff --git a/modoboa_installer/scripts/postfix.py b/modoboa_installer/scripts/postfix.py index 5c95c84..b6c6955 100644 --- a/modoboa_installer/scripts/postfix.py +++ b/modoboa_installer/scripts/postfix.py @@ -60,6 +60,8 @@ class Postfix(base.Installer): "modoboa", "venv_path"), "modoboa_instance_path": self.config.get( "modoboa", "instance_path"), + "opendkim_port": self.config.get( + "opendkim", "port") }) return context diff --git a/run.py b/run.py index d796bba..f370f9e 100755 --- a/run.py +++ b/run.py @@ -96,6 +96,7 @@ def main(input_args): scripts.install("radicale", config) scripts.install("uwsgi", config) scripts.install("nginx", config) + scripts.install("opendkim", config) scripts.install("postfix", config) scripts.install("dovecot", config) utils.printcolor( diff --git a/tests.py b/tests.py index 67068bd..4b30735 100644 --- a/tests.py +++ b/tests.py @@ -40,7 +40,7 @@ class ConfigFileTestCase(unittest.TestCase): def test_interactive_mode(self, mock_user_input): """Check interactive mode.""" mock_user_input.side_effect = [ - "0", "0", "", "", "", "" + "0", "0", "", "", "", "", "" ] with open(os.devnull, "w") as fp: sys.stdout = fp @@ -59,7 +59,7 @@ class ConfigFileTestCase(unittest.TestCase): def test_interactive_mode_letsencrypt(self, mock_user_input): """Check interactive mode.""" mock_user_input.side_effect = [ - "1", "admin@example.test", "0", "", "", "", "" + "1", "admin@example.test", "0", "", "", "", "", "" ] with open(os.devnull, "w") as fp: sys.stdout = fp From 0458b3013566d5506cd45159051baceca950e658 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 6 Apr 2018 11:50:11 +0200 Subject: [PATCH 51/56] Make sure the right caldav version is installed. --- modoboa_installer/scripts/modoboa.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index f27db93..9ac8645 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -91,6 +91,10 @@ class Modoboa(base.Installer): if sys.version_info.major == 2 and sys.version_info.micro < 9: # Add extra packages to fix the SNI issue packages += ["pyOpenSSL"] + if "modoboa-radicale" in self.extensions: + # Temp. fix + packages += [ + "https://github.com/modoboa/caldav/tarball/master#egg=caldav"] python.install_packages(packages, self.venv_path, sudo_user=self.user) if self.devmode: # FIXME: use dev-requirements instead From 557177ca4847fcd1cfcdbdaf807e44b70c25d568 Mon Sep 17 00:00:00 2001 From: Christophe CHAUVET Date: Sat, 7 Apr 2018 16:44:26 +0200 Subject: [PATCH 52/56] opendkim: add LogResults = Yes If logging is enabled (see Syslog below), requests that the results of evaluation of all signatures that were at least partly intact (i.e., the "d=", "s=", and "b=" tags could be extracted). --- modoboa_installer/scripts/files/opendkim/opendkim.conf.tpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/files/opendkim/opendkim.conf.tpl b/modoboa_installer/scripts/files/opendkim/opendkim.conf.tpl index 890d25c..9f689fc 100644 --- a/modoboa_installer/scripts/files/opendkim/opendkim.conf.tpl +++ b/modoboa_installer/scripts/files/opendkim/opendkim.conf.tpl @@ -4,8 +4,10 @@ # Log to syslog Syslog yes -LogWhy Yes SyslogSuccess Yes +LogWhy Yes +LogResults Yes + # Required to use local socket with MTAs that access the socket as a non- # privileged user (e.g. Postfix) UMask 007 From ff2bf9994d7b02707bef364732fb439db63cb81c Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 11 Apr 2018 14:12:24 +0200 Subject: [PATCH 53/56] Install modoboa from sources until we find a solution. see #197 --- modoboa_installer/python.py | 12 ++++++++---- modoboa_installer/scripts/modoboa.py | 8 ++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/modoboa_installer/python.py b/modoboa_installer/python.py index 3c07f19..fc89cf6 100644 --- a/modoboa_installer/python.py +++ b/modoboa_installer/python.py @@ -22,10 +22,14 @@ def get_pip_path(venv): return binpath -def install_package(name, venv=None, upgrade=False, **kwargs): +def install_package(name, venv=None, upgrade=False, binary=True, **kwargs): """Install a Python package using pip.""" - cmd = "{} install {}{}".format( - get_pip_path(venv), " -U " if upgrade else "", name) + cmd = "{} install{}{} {}".format( + get_pip_path(venv), + " -U" if upgrade else "", + " --no-binary :all:" if not binary else "", + name + ) utils.exec_cmd(cmd, **kwargs) @@ -67,4 +71,4 @@ def setup_virtualenv(path, sudo_user=None, python_version=2): utils.exec_cmd("virtualenv {}".format(path)) else: utils.exec_cmd("{} -m venv {}".format(python_binary, path)) - install_package("pip", venv=path, upgrade=True) + install_packages(["pip", "setuptools"], venv=path, upgrade=True) diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 9ac8645..a00a2c7 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -69,10 +69,11 @@ class Modoboa(base.Installer): packages = ["rrdtool"] version = self.config.get("modoboa", "version") if version == "latest": - packages += ["modoboa"] + self.extensions + modoboa_package = "modoboa" + packages += self.extensions else: matrix = compatibility_matrix.COMPATIBILITY_MATRIX[version] - packages.append("modoboa=={}".format(version)) + modoboa_package = "modoboa=={}".format(version) for extension in list(self.extensions): if not self.is_extension_ok_for_version(extension, version): self.extensions.remove(extension) @@ -84,6 +85,9 @@ class Modoboa(base.Installer): packages.append("{}{}".format(extension, req_version)) else: packages.append(extension) + # Temp fix for https://github.com/modoboa/modoboa-installer/issues/197 + python.install_package( + modoboa_package, self.venv_path, binary=False, sudo_user=self.user) if self.dbengine == "postgres": packages.append("psycopg2-binary") else: From fc64cd07e018a803a9d7de25c9d601eb98204105 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 9 May 2018 14:46:18 +0200 Subject: [PATCH 54/56] Quick fix for Ubuntu 18.04 compat. fix #211 --- modoboa_installer/scripts/amavis.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modoboa_installer/scripts/amavis.py b/modoboa_installer/scripts/amavis.py index 3f8ef7e..ed50ab5 100644 --- a/modoboa_installer/scripts/amavis.py +++ b/modoboa_installer/scripts/amavis.py @@ -1,5 +1,7 @@ """Amavis related functions.""" +import platform + from .. import package from .. import utils @@ -16,7 +18,7 @@ class Amavis(base.Installer): "deb": [ "libdbi-perl", "amavisd-new", "arc", "arj", "cabextract", "liblz4-tool", "lrzip", "lzop", "p7zip-full", "rpm2cpio", - "unrar-free", "zoo", "ripole" + "unrar-free", "ripole" ], "rpm": [ "amavisd-new", "arj", "cabextract", "lz4", "lrzip", @@ -50,6 +52,9 @@ class Amavis(base.Installer): """Additional packages.""" packages = super(Amavis, self).get_packages() if package.backend.FORMAT == "deb": + if platform.linux_distribution()[2] != "bionic": + # Quick fix + packages.append("zoo") db_driver = "pg" if self.db_driver == "pgsql" else self.db_driver return packages + ["libdbd-{}-perl".format(db_driver)] if self.db_driver == "pgsql": From a7373e364e0e85f9ad10405f79b726e8823ac90e Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 11 May 2018 12:02:01 +0200 Subject: [PATCH 55/56] Fixed dist detection. fix #212 --- modoboa_installer/database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modoboa_installer/database.py b/modoboa_installer/database.py index e1a5ed3..16cdb5e 100644 --- a/modoboa_installer/database.py +++ b/modoboa_installer/database.py @@ -144,6 +144,7 @@ class MySQL(Database): def install_package(self): """Preseed package installation.""" name, version, _id = platform.linux_distribution() + name = name.lower() if name == "debian": mysql_name = "mysql" if version.startswith("8") else "mariadb" self.packages["deb"].append("lib{}client-dev".format(mysql_name)) From ab1c8254e0ae603876cb21fa7f889dfaedc2021c Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 30 May 2018 16:45:00 +0200 Subject: [PATCH 56/56] Added missing configuration for opendkim. see #207 --- modoboa_installer/scripts/opendkim.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/modoboa_installer/scripts/opendkim.py b/modoboa_installer/scripts/opendkim.py index 6d904af..34bdebf 100644 --- a/modoboa_installer/scripts/opendkim.py +++ b/modoboa_installer/scripts/opendkim.py @@ -76,3 +76,18 @@ class Opendkim(base.Installer): ) self.backend.grant_right_on_table( dbname, "dkim", self.app_config["dbuser"], "SELECT") + + def post_run(self): + """Addtional tasks.""" + if package.backend.FORMAT != "deb": + return + pattern = ( + "s/^SOCKET=local:\$RUNDIR\/opendkim\.sock/" + "#SOCKET=local:\$RUNDIR\/opendkim\.sock/" + ) + utils.exec_cmd("perl -pi -e '{}' /etc/default/opendkim".format(pattern)) + pattern = ( + "s/^#SOCKET=inet:12345\@localhost$/" + "SOCKET=inet:12345\@localhost/" + ) + utils.exec_cmd("perl -pi -e '{}' /etc/default/opendkim".format(pattern))