From 5f36d6231ffca9d282201d14d0fda6b0188c116a Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Thu, 9 Feb 2017 14:40:39 +0100 Subject: [PATCH 1/3] Added automx support. see #98 --- README.rst | 4 +- installer.cfg | 14 ++- modoboa_installer/python.py | 8 ++ modoboa_installer/scripts/automx.py | 92 +++++++++++++++++++ modoboa_installer/scripts/base.py | 1 - .../scripts/files/automx/automx.conf.tpl | 38 ++++++++ .../scripts/files/nginx/automx.conf.tpl | 16 ++++ .../{nginx.conf.tpl => modoboa.conf.tpl} | 2 +- .../scripts/files/uwsgi/automx.ini.tpl | 14 +++ .../uwsgi/{uwsgi.ini.tpl => modoboa.ini.tpl} | 8 +- modoboa_installer/scripts/modoboa.py | 5 +- modoboa_installer/scripts/nginx.py | 30 +++--- modoboa_installer/scripts/uwsgi.py | 80 +++++++++++----- run.py | 15 +-- 14 files changed, 273 insertions(+), 54 deletions(-) create mode 100644 modoboa_installer/scripts/automx.py create mode 100644 modoboa_installer/scripts/files/automx/automx.conf.tpl create mode 100644 modoboa_installer/scripts/files/nginx/automx.conf.tpl rename modoboa_installer/scripts/files/nginx/{nginx.conf.tpl => modoboa.conf.tpl} (97%) create mode 100644 modoboa_installer/scripts/files/uwsgi/automx.ini.tpl rename modoboa_installer/scripts/files/uwsgi/{uwsgi.ini.tpl => modoboa.ini.tpl} (66%) diff --git a/README.rst b/README.rst index be8a13f..f556685 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ Usage:: $ git clone https://github.com/modoboa/modoboa-installer $ cd modoboa-installer - $ sudo ./run.py + $ sudo ./run.py To customize the installation, look at the ``installer.cfg`` file. @@ -35,7 +35,7 @@ By default, the following components are installed: * Nginx and uWSGI * Postfix * Dovecot -* Amavis (with SpamAssassin and ClamAV) +* Amavis (with SpamAssassin and ClamAV) If you want more information about the installation process, add the ``--debug`` option to your command line. diff --git a/installer.cfg b/installer.cfg index 34fccf5..d67b706 100644 --- a/installer.cfg +++ b/installer.cfg @@ -1,3 +1,7 @@ +[general] +# %(domain)s is the value specified when launching the installer +hostname = mail.%(domain)s + [certificate] generate = true # Choose between self-signed or letsencrypt @@ -39,6 +43,14 @@ extensions = modoboa-amavis modoboa-pdfcredentials modoboa-postfix-autoreply mod # Deploy Modoboa and enable development mode devmode = false +[automx] +enabled = true +user = automx +config_dir = /etc +home_dir = /srv/automx +venv_path = %(home_dir)s/env +instance_path = %(home_dir)s/instance + [amavis] enabled = true user = amavis @@ -86,6 +98,6 @@ dbuser = spamassassin dbpassword = password [uwsgi] -enabled = true +enabled = true config_dir = /etc/uwsgi nb_processes = 2 diff --git a/modoboa_installer/python.py b/modoboa_installer/python.py index f8c0fbb..87edd93 100644 --- a/modoboa_installer/python.py +++ b/modoboa_installer/python.py @@ -6,6 +6,14 @@ from . import package from . import utils +def get_path(cmd, venv=None): + """Return path to cmd.""" + path = cmd + if venv: + path = os.path.join(venv, "bin", path) + return path + + def get_pip_path(venv): """Return the full path to pip command.""" binpath = "pip" diff --git a/modoboa_installer/scripts/automx.py b/modoboa_installer/scripts/automx.py new file mode 100644 index 0000000..21754de --- /dev/null +++ b/modoboa_installer/scripts/automx.py @@ -0,0 +1,92 @@ +"""Automx related tasks.""" + +import os +import pwd +import shutil +import stat + +from .. import python +from .. import utils + +from . import base + + +class Automx(base.Installer): + """Automx installation.""" + + appname = "automx" + config_files = ["automx.conf"] + no_daemon = True + packages = { + "deb": ["memcached", "unzip"], + "rpm": [] + } + with_user = True + + def __init__(self, config): + """Get configuration.""" + super(Automx, self).__init__(config) + self.venv_path = config.get("automx", "venv_path") + self.instance_path = config.get("automx", "instance_path") + + def get_template_context(self): + """Additional variables.""" + context = super(Automx, self).get_template_context() + sql_dsn = "{}://{}:{}@{}/{}".format( + self.dbengine, + self.config.get("modoboa", "dbuser"), + self.config.get("modoboa", "dbpassword"), + self.dbhost, + self.config.get("modoboa", "dbname")) + if self.db_driver == "pgsql": + sql_query = ( + "SELECT first_name || ' ' || last_name AS display_name, email " + "FROM core_user WHERE email='%s' AND is_active") + else: + sql_query = ( + "SELECT concat(first_name, ' ', last_name) AS display_name, " + "email FROM core_user WHERE email='%s' AND is_active=1" + ) + context.update({"sql_dsn": sql_dsn, "sql_query": sql_query}) + return context + + def _setup_venv(self): + """Prepare a python virtualenv.""" + python.setup_virtualenv(self.venv_path, sudo_user=self.user) + packages = [ + "future", "lxml", "ipaddress", "sqlalchemy", "python-memcached", + "python-dateutil", "configparser" + ] + python.install_packages(packages, self.venv_path, sudo_user=self.user) + target = "{}/master.zip".format(self.home_dir) + if os.path.exists(target): + os.unlink(target) + utils.exec_cmd( + "wget https://github.com/sys4/automx/archive/master.zip", + sudo_user=self.user, cwd=self.home_dir) + self.repo_dir = "{}/automx-master".format(self.home_dir) + if os.path.exists(self.repo_dir): + shutil.rmtree(self.repo_dir) + utils.exec_cmd( + "unzip master.zip", sudo_user=self.user, cwd=self.home_dir) + utils.exec_cmd( + "{} setup.py install".format( + python.get_path("python", self.venv_path)), + cwd=self.repo_dir) + + def _deploy_instance(self): + """Copy files to instance dir.""" + if not os.path.exists(self.instance_path): + pw = pwd.getpwnam(self.user) + mode = ( + stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + utils.mkdir(self.instance_path, mode, pw[2], pw[3]) + path = "{}/src/automx_wsgi.py".format(self.repo_dir) + utils.exec_cmd("cp {} {}".format(path, self.instance_path), + sudo_user=self.user, cwd=self.home_dir) + + def post_run(self): + """Additional tasks.""" + self._setup_venv() + self._deploy_instance() diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 74c2e04..6ec63b6 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -9,7 +9,6 @@ from .. import utils class Installer(object): - """Simple installer for one application.""" appname = None diff --git a/modoboa_installer/scripts/files/automx/automx.conf.tpl b/modoboa_installer/scripts/files/automx/automx.conf.tpl new file mode 100644 index 0000000..7f66272 --- /dev/null +++ b/modoboa_installer/scripts/files/automx/automx.conf.tpl @@ -0,0 +1,38 @@ +[automx] +provider = %domain +domains = * + +# Protect against DoS +memcache = 127.0.0.1:11211 +memcache_ttl = 600 +client_error_limit = 20 +rate_limit_exception_networks = 127.0.0.0/8, ::1/128 + +[global] +backend = sql +actions = settings +account_type = email +host = %sql_dsn +query = %sql_query +result_attrs = display_name, email + +smtp = yes +smtp_server = %hostname +smtp_port = 587 +smtp_encryption = starttls +smtp_auth = plaintext +smtp_refresh_ttl = 6 +smtp_default = yes + +imap = yes +imap_server = %hostname +imap_port = 143 +imap_encryption = starttls +imap_auth = plaintext +imap_refresh_ttl = 6 + +pop = yes +pop_server = %hostname +pop_port = 110 +pop_encryption = starttls +pop_auth = plaintext diff --git a/modoboa_installer/scripts/files/nginx/automx.conf.tpl b/modoboa_installer/scripts/files/nginx/automx.conf.tpl new file mode 100644 index 0000000..96ec266 --- /dev/null +++ b/modoboa_installer/scripts/files/nginx/automx.conf.tpl @@ -0,0 +1,16 @@ +upstream automx { + server unix:%uwsgi_socket_path fail_timeout=0; +} + +server { + listen 80; + server_name %hostname; + + access_log /var/log/nginx/%{hostname}-access.log; + error_log /var/log/nginx/%{hostname}-error.log; + + location /mail/config-v1.1.xml { + include uwsgi_params; + uwsgi_pass automx; + } +} \ No newline at end of file diff --git a/modoboa_installer/scripts/files/nginx/nginx.conf.tpl b/modoboa_installer/scripts/files/nginx/modoboa.conf.tpl similarity index 97% rename from modoboa_installer/scripts/files/nginx/nginx.conf.tpl rename to modoboa_installer/scripts/files/nginx/modoboa.conf.tpl index 98563db..ac23191 100644 --- a/modoboa_installer/scripts/files/nginx/nginx.conf.tpl +++ b/modoboa_installer/scripts/files/nginx/modoboa.conf.tpl @@ -11,7 +11,7 @@ server { server { listen 443 ssl; server_name %hostname; - root %modoboa_instance_path; + root %app_instance_path; ssl_certificate %tls_cert_file; ssl_certificate_key %tls_key_file; diff --git a/modoboa_installer/scripts/files/uwsgi/automx.ini.tpl b/modoboa_installer/scripts/files/uwsgi/automx.ini.tpl new file mode 100644 index 0000000..f6ab23c --- /dev/null +++ b/modoboa_installer/scripts/files/uwsgi/automx.ini.tpl @@ -0,0 +1,14 @@ +[uwsgi] +uid = %app_user +gid = %app_user +plugins = python +home = %app_venv_path +chdir = %app_instance_path +module = automx_wsgi +master = true +vhost = true +harakiri = 60 +processes = %nb_processes +socket = %uwsgi_socket_path +chmod-socket = 660 +vacuum = true diff --git a/modoboa_installer/scripts/files/uwsgi/uwsgi.ini.tpl b/modoboa_installer/scripts/files/uwsgi/modoboa.ini.tpl similarity index 66% rename from modoboa_installer/scripts/files/uwsgi/uwsgi.ini.tpl rename to modoboa_installer/scripts/files/uwsgi/modoboa.ini.tpl index a013085..a9f431e 100644 --- a/modoboa_installer/scripts/files/uwsgi/uwsgi.ini.tpl +++ b/modoboa_installer/scripts/files/uwsgi/modoboa.ini.tpl @@ -1,9 +1,9 @@ [uwsgi] -uid = %modoboa_user -gid = %modoboa_user +uid = %app_user +gid = %app_user plugins = python -home = %modoboa_venv_path -chdir = %modoboa_instance_path +home = %app_venv_path +chdir = %app_instance_path module = instance.wsgi:application master = true processes = %nb_processes diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index 80e9a7b..ff6ed6a 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -11,7 +11,6 @@ from .. import python from .. import utils from . import base -from . import install class Modoboa(base.Installer): @@ -44,7 +43,7 @@ class Modoboa(base.Installer): self.devmode = config.getboolean("modoboa", "devmode") def _setup_venv(self): - """Prepare a dedicated virtuelenv.""" + """Prepare a dedicated virtualenv.""" python.setup_virtualenv(self.venv_path, sudo_user=self.user) packages = ["modoboa", "rrdtool"] if self.dbengine == "postgres": @@ -171,5 +170,3 @@ class Modoboa(base.Installer): self._setup_venv() self._deploy_instance() self.apply_settings() - install("uwsgi", self.config) - install("nginx", self.config) diff --git a/modoboa_installer/scripts/nginx.py b/modoboa_installer/scripts/nginx.py index 6ea42e9..f1d5f9f 100644 --- a/modoboa_installer/scripts/nginx.py +++ b/modoboa_installer/scripts/nginx.py @@ -11,7 +11,6 @@ from .uwsgi import Uwsgi class Nginx(base.Installer): - """Nginx installer.""" appname = "nginx" @@ -20,21 +19,23 @@ class Nginx(base.Installer): "rpm": ["nginx"] } - def get_template_context(self): + def get_template_context(self, app): """Additionnal variables.""" context = super(Nginx, self).get_template_context() context.update({ - "modoboa_instance_path": ( - self.config.get("modoboa", "instance_path")), - "uwsgi_socket_path": Uwsgi(self.config).socket_path + "app_instance_path": ( + self.config.get(app, "instance_path")), + "uwsgi_socket_path": Uwsgi(self.config).get_socket_path(app) }) return context - def post_run(self): - """Additionnal tasks.""" - hostname = self.config.get("general", "hostname") - context = self.get_template_context() - src = self.get_file_path("nginx.conf.tpl") + def _setup_config(self, app, hostname=None): + """Custom app configuration.""" + if hostname is None: + hostname = self.config.get("general", "hostname") + context = self.get_template_context(app) + context.update({"hostname": hostname}) + src = self.get_file_path("{}.conf.tpl".format(app)) if package.backend.FORMAT == "deb": dst = os.path.join( self.config_dir, "sites-available", "{}.conf".format(hostname)) @@ -44,7 +45,7 @@ class Nginx(base.Installer): if os.path.exists(link): return os.symlink(dst, link) - group = self.config.get("modoboa", "user") + group = self.config.get(app, "user") user = "www-data" else: dst = os.path.join( @@ -54,6 +55,13 @@ class Nginx(base.Installer): user = "nginx" system.add_user_to_group(user, group) + def post_run(self): + """Additionnal tasks.""" + self._setup_config("modoboa") + if self.config.getboolean("automx", "enabled"): + hostname = "autoconfig.{}".format( + self.config.get("general", "domain")) + self._setup_config("automx", hostname) if not os.path.exists("{}/dhparam.pem".format(self.config_dir)): cmd = "openssl dhparam -dsaparam -out dhparam.pem 4096" utils.exec_cmd(cmd, cwd=self.config_dir) diff --git a/modoboa_installer/scripts/uwsgi.py b/modoboa_installer/scripts/uwsgi.py index 9014985..4885d62 100644 --- a/modoboa_installer/scripts/uwsgi.py +++ b/modoboa_installer/scripts/uwsgi.py @@ -10,7 +10,6 @@ from . import base class Uwsgi(base.Installer): - """uWSGI installer.""" appname = "uwsgi" @@ -19,22 +18,21 @@ class Uwsgi(base.Installer): "rpm": ["uwsgi", "uwsgi-plugin-python"], } - @property - def socket_path(self): + def get_socket_path(self, app): """Return socket path.""" if package.backend.FORMAT == "deb": - return "/run/uwsgi/app/modoboa_instance/socket" - return "/run/uwsgi/modoboa_instance.sock" + return "/run/uwsgi/app/{}_instance/socket".format(app) + return "/run/uwsgi/{}_instance.sock".format(app) - def get_template_context(self): + def get_template_context(self, app): """Additionnal variables.""" context = super(Uwsgi, self).get_template_context() context.update({ - "modoboa_user": self.config.get("modoboa", "user"), - "modoboa_venv_path": self.config.get("modoboa", "venv_path"), - "modoboa_instance_path": ( - self.config.get("modoboa", "instance_path")), - "uwsgi_socket_path": self.socket_path, + "app_user": self.config.get(app, "user"), + "app_venv_path": self.config.get(app, "venv_path"), + "app_instance_path": ( + self.config.get(app, "instance_path")), + "uwsgi_socket_path": self.get_socket_path(app), }) return context @@ -44,18 +42,28 @@ class Uwsgi(base.Installer): return os.path.join(self.config_dir, "apps-available") return "{}.d".format(self.config_dir) - def post_run(self): - """Additionnal tasks.""" - context = self.get_template_context() - src = self.get_file_path("uwsgi.ini.tpl") - dst = os.path.join(self.get_config_dir(), "modoboa_instance.ini") + def _enable_config_debian(self, dst): + """Enable config file.""" + link = os.path.join( + self.config_dir, "apps-enabled", os.path.basename(dst)) + if os.path.exists(link): + return + os.symlink(dst, link) + + def _setup_config(self, app): + """Common setup code.""" + context = self.get_template_context(app) + src = self.get_file_path("{}.ini.tpl".format(app)) + dst = os.path.join( + self.get_config_dir(), "{}_instance.ini".format(app)) utils.copy_from_template(src, dst, context) + return dst + + def _setup_modoboa_config(self): + """Custom modoboa configuration.""" + dst = self._setup_config("modoboa") if package.backend.FORMAT == "deb": - link = os.path.join( - self.config_dir, "apps-enabled", os.path.basename(dst)) - if os.path.exists(link): - return - os.symlink(dst, link) + self._enable_config_debian(dst) else: system.add_user_to_group( "uwsgi", self.config.get("modoboa", "user")) @@ -68,8 +76,32 @@ class Uwsgi(base.Installer): utils.exec_cmd( "perl -pi -e '{}' /etc/uwsgi.ini".format(pattern)) + def _setup_automx_config(self): + """Custom automx configuration.""" + dst = self._setup_config("automx") + if package.backend.FORMAT == "deb": + self._enable_config_debian(dst) + else: + system.add_user_to_group( + "uwsgi", self.config.get("automx", "user")) + pattern = ( + "s/emperor-tyrant = true/emperor-tyrant = false/") + utils.exec_cmd( + "perl -pi -e '{}' /etc/uwsgi.ini".format(pattern)) + + def post_run(self): + """Additionnal tasks.""" + self._setup_modoboa_config() + if self.config.getboolean("automx", "enabled"): + self._setup_automx_config() + def restart_daemon(self): """Restart daemon process.""" - code, output = utils.exec_cmd("service uwsgi status modoboa_instance") - action = "start" if code else "restart" - utils.exec_cmd("service uwsgi {}".format(action)) + instances = ["modoboa_instance"] + if self.config.getboolean("automx", "enabled"): + instances.append("automx_instance") + for instance in instances: + code, output = utils.exec_cmd("service uwsgi status {}".format( + instance)) + action = "start" if code else "restart" + utils.exec_cmd("service uwsgi {}".format(action)) diff --git a/run.py b/run.py index 0fc99c7..a1de050 100755 --- a/run.py +++ b/run.py @@ -21,8 +21,8 @@ def main(): help="Enable debug output") parser.add_argument("--force", action="store_true", default=False, help="Force installation") - parser.add_argument("hostname", type=str, - help="The hostname of your future mail server") + parser.add_argument("domain", type=str, + help="The main domain of your future mail server") args = parser.parse_args() if args.debug: @@ -33,10 +33,10 @@ def main(): config.readfp(fp) if not config.has_section("general"): config.add_section("general") - config.set("general", "hostname", args.hostname) + config.set("general", "domain", args.domain) utils.printcolor( - "Your mail server {} will be installed with the following components:" - .format(args.hostname), utils.BLUE) + "Your mail server will be installed with the following components:", + utils.BLUE) components = [] for section in config.sections(): if section in ["general", "database", "mysql", "postgres", @@ -61,12 +61,15 @@ def main(): if ssl_backend: ssl_backend.create() scripts.install("modoboa", config) + scripts.install("automx", config) + scripts.install("uwsgi", config) + scripts.install("nginx", config) scripts.install("postfix", config) scripts.install("amavis", config) scripts.install("dovecot", config) utils.printcolor( "Congratulations! You can enjoy Modoboa at https://{} (admin:password)" - .format(args.hostname), + .format(config.get("general", "hostname")), utils.GREEN) if __name__ == "__main__": From 879176605221aa5976040afb98a5a32da705bee1 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Thu, 9 Feb 2017 16:31:14 +0100 Subject: [PATCH 2/3] CentOS support and enabled memcached at startup. see #98 --- modoboa_installer/scripts/automx.py | 4 +++- modoboa_installer/scripts/base.py | 5 +---- modoboa_installer/system.py | 8 ++++++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/modoboa_installer/scripts/automx.py b/modoboa_installer/scripts/automx.py index 21754de..e278645 100644 --- a/modoboa_installer/scripts/automx.py +++ b/modoboa_installer/scripts/automx.py @@ -6,6 +6,7 @@ import shutil import stat from .. import python +from .. import system from .. import utils from . import base @@ -19,7 +20,7 @@ class Automx(base.Installer): no_daemon = True packages = { "deb": ["memcached", "unzip"], - "rpm": [] + "rpm": ["memcached", "unzip"] } with_user = True @@ -90,3 +91,4 @@ class Automx(base.Installer): """Additional tasks.""" self._setup_venv() self._deploy_instance() + system.enable_and_start_service("memcached") diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 6ec63b6..7fdb89a 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -136,10 +136,7 @@ class Installer(object): if self.no_daemon: return name = self.get_daemon_name() - system.enable_service(name) - code, output = utils.exec_cmd("service {} status".format(name)) - action = "start" if code else "restart" - utils.exec_cmd("service {} {}".format(name, action)) + system.enable_and_start_service(name) def run(self): """Run the installer.""" diff --git a/modoboa_installer/system.py b/modoboa_installer/system.py index 968d7a3..be6a970 100644 --- a/modoboa_installer/system.py +++ b/modoboa_installer/system.py @@ -49,3 +49,11 @@ def add_user_to_group(user, group): def enable_service(name): """Enable a service at startup.""" utils.exec_cmd("systemctl enable {}".format(name)) + + +def enable_and_start_service(name): + """Enable a start a service.""" + enable_service(name) + code, output = utils.exec_cmd("service {} status".format(name)) + action = "start" if code else "restart" + utils.exec_cmd("service {} {}".format(name, action)) From 6fd517e2df0e5335692f77a07280668034621b23 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Fri, 10 Feb 2017 11:43:19 +0100 Subject: [PATCH 3/3] Updated config. file see #98 --- modoboa_installer/scripts/files/automx/automx.conf.tpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modoboa_installer/scripts/files/automx/automx.conf.tpl b/modoboa_installer/scripts/files/automx/automx.conf.tpl index 7f66272..d69a792 100644 --- a/modoboa_installer/scripts/files/automx/automx.conf.tpl +++ b/modoboa_installer/scripts/files/automx/automx.conf.tpl @@ -21,6 +21,7 @@ smtp_server = %hostname smtp_port = 587 smtp_encryption = starttls smtp_auth = plaintext +smtp_auth_identity = ${email} smtp_refresh_ttl = 6 smtp_default = yes @@ -29,6 +30,7 @@ imap_server = %hostname imap_port = 143 imap_encryption = starttls imap_auth = plaintext +imap_auth_identity = ${email} imap_refresh_ttl = 6 pop = yes @@ -36,3 +38,4 @@ pop_server = %hostname pop_port = 110 pop_encryption = starttls pop_auth = plaintext +pop_auth_identity = ${email}