From 6b4302b566d9a75c2fa76d0c82e6534de9aaebdb Mon Sep 17 00:00:00 2001 From: Spitap Date: Thu, 22 Dec 2022 18:47:23 +0100 Subject: [PATCH] Update from master commit 5c22600d98a19cedc7ce385c93e5ed998c639e6e Merge: bc12ca7 bcdbb4a Author: Antoine Nguyen Date: Tue Nov 29 16:54:28 2022 +0100 Merge pull request #462 from Spitfireap/randomize-api-call-time randomize api call time commit bcdbb4a2ce658ca6a01409dbc89b437b74fdcd5e Author: Spitap Date: Tue Nov 29 14:53:05 2022 +0100 fix typo commit bd1ddcef217045c1c998daa83dc31d9b7dfe319f Author: Spitap Date: Tue Nov 29 13:45:31 2022 +0100 randomize api call time commit bc12ca7327fd2f17503f5c331df5502b2f5932fb Merge: d364239 bd0ecd0 Author: Antoine Nguyen Date: Mon Nov 14 15:49:41 2022 +0100 Merge pull request #458 from Spitfireap/fix-include_try fix typo in dovecot configuration file commit bd0ecd09492a8cd9703f4cde95d51b9ccfc6f1ca Author: Spitap Date: Thu Nov 10 14:57:43 2022 +0100 fix typo in dovecot configuration file commit d364239348450c58ab7bae1b7eb990a24fdaa420 Merge: 61838db 3763300 Author: Antoine Nguyen Date: Wed Nov 9 10:51:30 2022 +0100 Merge pull request #456 from modoboa/feature/improved_backup_restore WIP: Improved backup/restore system. commit 37633008cb05ebfe9c617c848e119d67fdd32e7e Author: Antoine Nguyen Date: Wed Nov 9 10:30:44 2022 +0100 Fixed restore mode commit d6f9a5b913aeca75bfc34e67c6787958334e08d2 Author: Antoine Nguyen Date: Tue Nov 8 17:20:25 2022 +0100 Few fixes. commit 8b1d60ee596b57fc3dd0b3de21b7a6f82b9f4899 Author: Antoine Nguyen Date: Tue Nov 8 17:19:23 2022 +0100 Few fixes commit 2b5edae5d53ee079295d859e47a0b2626084e001 Author: Antoine Nguyen Date: Sun Nov 6 10:30:24 2022 +0100 WIP: Improved backup/restore system. commit 61838dbe4d8d6121fa4f14bf6a0d254fc7229292 Author: Antoine Nguyen Date: Sat Nov 5 09:30:50 2022 +0100 Check if restore is defined before doing anything else. fix #453 commit 962cac3ad97139fdaab98facdfa985da1f8c958d Merge: 1b192c5 ef2359a Author: Antoine Nguyen Date: Fri Nov 4 09:41:20 2022 +0100 Merge pull request #450 from Spitfireap/fixed-super-call fixed super call in modoboa's script commit ef2359a2a88f61e4adcc44e09d02cb078796ada9 Author: Spitap Date: Thu Nov 3 23:10:21 2022 +0100 fixed super call commit 1b192c5fd5422351440f1a264d3d6356123d2e8a Merge: 754d652 b0b0146 Author: Antoine Nguyen Date: Thu Nov 3 15:34:48 2022 +0100 Merge pull request #449 from Spitfireap/fixed-import-typo fixed constants import commit b0b01465d9397c60467fa6938cd87599cccf5cbf Author: Spitap Date: Thu Nov 3 15:00:07 2022 +0100 fixed constants import commit 754d652fc240f9985dd376653d8139454d189053 Author: Antoine Nguyen Date: Thu Nov 3 12:27:04 2022 +0100 Few fixes commit cb5fa7569349080cd59b8a339ce855047f4b6bd4 Merge: 1afb8e6 e01265a Author: Antoine Nguyen Date: Thu Nov 3 12:20:25 2022 +0100 Merge pull request #444 from Spitfireap/tighter-config-file-perm tighter config file permission commit 1afb8e61fc53c760bf24097afb1f1955827f28f7 Merge: 15c1779 8dd0b7d Author: Antoine Nguyen Date: Thu Nov 3 12:17:16 2022 +0100 Merge pull request #424 from Spitfireap/restore Backup & restore system commit 8dd0b7d4975e9fdbc2f7ae50622072174441b4ed Author: Spitap Date: Thu Nov 3 10:57:03 2022 +0100 Last camelCase commit 554611b36603115f5f42f59874ac2bf36b181ed2 Author: Spitap Date: Thu Nov 3 10:54:06 2022 +0100 review fix commit 15c17796f2d91155fa7ab721b35d2a6d9be20ebc Merge: ce8e7e6 84d1363 Author: Antoine Nguyen Date: Fri Oct 28 09:43:30 2022 +0200 Merge pull request #446 from Spitfireap/fix-ssl-min-protocol fixed ssl_min_protocol setting commit 84d13633a16c149e00e13c414e2fd846318d5146 Author: Spitap Date: Thu Oct 27 22:37:47 2022 +0200 fixed ssl_min_protocol setting commit ce8e7e602796efb3b7061539662ffdc0059ccf0c Merge: 8e8ae5f fe7df27 Author: Antoine Nguyen Date: Thu Oct 27 17:56:37 2022 +0200 Merge pull request #445 from Spitfireap/dovecot-fixes Fixes ssl permission error, updated ssl_protocol parameter commit e01265a4ee7b6ae3871d87ec5cd99826c10765e3 Merge: a5fba03 235ef3b Author: Spitap Date: Thu Oct 27 17:44:37 2022 +0200 Merge branch 'tighter-config-file-perm' of https://github.com/Spitfireap/modoboa-installer into tighter-config-file-perm commit a5fba032640ad09e678f6c76fdb4f5f713b82160 Author: Spitap Date: Thu Oct 27 11:13:47 2022 +0200 tighter config file permission commit fe7df276fc9cebafaf0684428c22ffb956eee108 Author: Spitap Date: Thu Oct 27 17:25:39 2022 +0200 Check dovecot version greater commit 8f34f0af6f8b4dbb7152f98774abd318b1660209 Author: Spitap Date: Thu Oct 27 17:00:58 2022 +0200 Fixes ssl permission error, updated ssl_protocol parameter commit 8e8ae5fb9c4750c246c85db44649051ff9c08413 Merge: 67f6cee fefbf54 Author: Antoine Nguyen Date: Thu Oct 27 16:49:20 2022 +0200 Merge pull request #439 from stefaweb/master Update config_dict_template.py for default max_servers value commit 235ef3befbdd4a2d26874b172c7a695fbaab2dc6 Author: Spitap Date: Thu Oct 27 11:13:47 2022 +0200 thighter config file permission commit 67f6cee8ea42a52e1b98c60015f2b3c174e99da9 Merge: b84abbb 53f7f8e Author: Antoine Nguyen Date: Tue Oct 25 19:32:37 2022 +0200 Merge pull request #442 from Spitfireap/patch-1 Set $max_server to 2 to avoid amavis crash commit 5c9d5c9a0319a9e94b9abd1906a09693cc7615d6 Author: Spitap Date: Tue Oct 25 16:58:57 2022 +0200 DKIM keys restore, Radicale backup/restore, fixes commit 4c1f8710b5600799027e4908470655a9478ad4a5 Author: Spitap Date: Tue Oct 25 16:04:55 2022 +0200 Added dkim key backup commit e34eb4b3372087844e444b23fe5043841f37d1c2 Author: Spitap Date: Tue Oct 25 13:59:28 2022 +0200 fix database path commit 53f7f8ef9d67d18f58e9f22a629bc1ed908820b5 Author: Spitfireap <45575529+Spitfireap@users.noreply.github.com> Date: Wed Oct 19 08:19:40 2022 +0000 Update config_dict_template.py commit 35778cd61448a0cbab6895cadb222edac2a32794 Merge: 6726f5b b84abbb Author: Spitfireap <45575529+Spitfireap@users.noreply.github.com> Date: Tue Oct 18 17:17:48 2022 +0200 Merge branch 'modoboa:master' into restore commit fefbf549a4db16faebce7cd76d6385cb2fa35f66 Author: Stephane Leclerc Date: Thu Oct 6 13:36:13 2022 +0200 Update config_dict_template.py for default max_server value commit 6726f5b1a232d30438b6a5ea175141523d1eb6c4 Author: Spitap Date: Mon Sep 26 13:39:28 2022 +0200 Improved path generation, path mistake proofing commit a192cbcbd0ab2b6169a66951fe0a44d32f210a33 Author: Spitap Date: Mon Sep 19 16:40:25 2022 +0200 Updated doc, default path on conf file commit 5bed9655ea2e75873846d562c4c8aefd00572a9a Author: Spitap Date: Mon Sep 19 15:53:19 2022 +0200 fixed typo commit 6b096a7470f07c77cf2ad34c7d5a0a9f518ea9c8 Author: Spitap Date: Mon Sep 19 15:50:03 2022 +0200 Simplified db dumps restore commit e30add03fdf59c23759aac07d8726bc692e06902 Merge: d75d83f 1f8dd1b Author: Spitap Date: Mon Sep 19 15:39:05 2022 +0200 Update from master commit d75d83f202811a00a948d56211ab480f158e5b23 Author: Spitap Date: Mon Sep 19 15:13:44 2022 +0200 more refactoring commit f3811b4b39c2f9e11e29d80895968402a17c42be Author: Spitap Date: Mon Sep 19 14:59:43 2022 +0200 refactoring commit b0d56b3989382aa6075a9719b1f9e8d50842b6dd Author: Spitap Date: Thu Sep 15 11:32:57 2022 +0200 PEP formating commit 53e3e3ec5853fe19de26e8de2d00adc8e5ee909d Author: Spitap Date: Fri Aug 5 15:20:11 2022 +0200 Better UX, use of os to concatenate path commit e546d2cb23f36385d379aee8efe29a72c4981b23 Author: Spitap Date: Wed Jul 27 16:32:59 2022 +0200 Better UX commit 70faa1c5cb3f1e156740d0789c66e0f72c929a08 Author: Spitap Date: Wed Jul 27 15:58:41 2022 +0200 Fixed backupdir index commit 563979a7ddc8bb5641f2349d59199e220c04c402 Author: Spitap Date: Wed Jul 27 15:51:22 2022 +0200 fixed mail backup/restore commit ee2ccf06474fb76c5b8fb6ae5b233eeed78319b3 Author: Spitap Date: Wed Jul 27 14:35:48 2022 +0200 Fixed postfix install, added restore to readme commit 2077c94b52c94e55328da3421cf3f378cada0e8b Author: Spitap Date: Tue Jul 26 17:05:00 2022 +0200 Fix amavis config file not copied to right location commit 4a7222bd2481a948b7535101608f78370fd567f4 Author: Spitap Date: Tue Jul 26 16:53:24 2022 +0200 Fixed nginx call to uwsgi commit e7b6104195b21bd6fc990bdc70981ada21a238a1 Author: Spitap Date: Tue Jul 26 16:39:41 2022 +0200 fixed install within class commit 4a00590354df5b21793b75734c11dbfdca6ffa6a Author: Spitap Date: Tue Jul 26 16:20:03 2022 +0200 fixed restore disclamer commit 15768c429e6eccd42a9eed21fe0845d252099ce3 Author: Spitap Date: Tue Jul 26 12:07:42 2022 +0200 Restore workflow done commit 439ffb94c44dec6b067e80e3aeea608a17aca796 Author: Spitap Date: Mon Jul 25 18:54:47 2022 +0200 initial commit commit 37bc21dfd34d1445d8d4b3b00744e231edc346c9 Author: Spitap Date: Tue Jul 26 10:36:08 2022 +0200 Backup postewhite.conf instead of custom whitelist Postwhite.conf contains a custom host list commit 26204143affc10a3e9bda60c52d259ef59ebd6e5 Merge: 2097055 d495afd Author: Spitap Date: Mon Jul 25 22:10:26 2022 +0200 Merge branch 'master' into backup commit 20970557de951c91b2af969eef52f221e58d60dd Author: Spitap Date: Mon Jul 25 22:05:35 2022 +0200 Allow to disable mail backup commit 632c26596e62e6219a485dfb1565cc369c23be87 Author: Spitap Date: Mon Jul 25 21:52:15 2022 +0200 Update backup readme commit 9e1c18cd6b19d267f126de1e984e0e992005696a Author: Spitap Date: Thu Jul 21 19:09:53 2022 +0200 Fix argument passed as list instead of string commit db6457c5f526a93c2b07153e606a23f18d60bcbe Author: Spitap Date: Thu Jul 21 19:07:18 2022 +0200 better path handling commit 579faccfa53e5a56cd7ce0106fbd4c6e4cbd28b1 Author: Spitap Date: Thu Jul 21 19:00:32 2022 +0200 added an automatic bash option (no path provided) or a path provided bash (for cron job) commit 5318fa279bcfdb7f930e7ed12ddd737cb77e6ae0 Author: Spitap Date: Thu Jul 21 18:00:50 2022 +0200 bash option commit 74de6a9bb1e4ee84abb23f2f1780cacd9c6c767f Author: Spitap Date: Thu Jul 21 17:31:56 2022 +0200 Reset pgpass before trying to backup secondary dbs commit 54185a7c5aa334c46924a037f018821c39596462 Author: Spitap Date: Thu Jul 21 17:26:40 2022 +0200 Fix database backup logic issue commit 1f9d69c37c28375774085293a417538deb66e87d Author: Spitap Date: Thu Jul 21 17:21:59 2022 +0200 Fix copy issue commit 8d02d2a9fb06acb9ef9590b6ee5fd6b6c9ba8f11 Author: Spitap Date: Thu Jul 21 17:09:23 2022 +0200 added safe mkdir in utils, use utils.mkdir_safe() in backup commit 6f604a5fec371d27d2383c95b4802cfedca79f07 Author: Spitap Date: Thu Jul 21 16:53:56 2022 +0200 Fix loop logic commit 568c4a65a08fab3bdf1e7b3dc190eaf86204f531 Author: Spitap Date: Thu Jul 21 16:51:32 2022 +0200 fix none-type passed to os.path commit dc84a7952801deeb4b863cd501ce7171120b8d31 Author: Spitap Date: Thu Jul 21 14:12:35 2022 +0200 Note : capitalize affects only first letter commit 304e25fa3c3da042a0803fe863829a23263a5d8e Author: Spitap Date: Thu Jul 21 14:10:57 2022 +0200 Fix getattr commit 070efd61c4b767ffa8c347c9d18271223317604d Author: Spitap Date: Thu Jul 21 14:08:39 2022 +0200 Fix import commit 9917d8023ec370caa1021937f0c7bb15d2371e49 Author: Spitap Date: Thu Jul 21 14:02:41 2022 +0200 Edited README, fix backup run process commit 27b9de6755c077c53662085a34019a0fa0310943 Author: Spitap Date: Thu Jul 21 13:48:44 2022 +0200 database backup commit 56ed214fb5e4ff039384c2e448667c52ba538aad Author: Spitap Date: Tue Jul 19 19:06:53 2022 +0200 Starting work on backup system --- .github/workflows/installer.yml | 60 +++++ .travis.yml | 15 -- README.rst | 53 +++- modoboa_installer/config_dict_template.py | 13 +- modoboa_installer/constants.py | 1 + modoboa_installer/database.py | 15 ++ modoboa_installer/scripts/__init__.py | 45 +++- modoboa_installer/scripts/amavis.py | 29 ++- modoboa_installer/scripts/backup.py | 231 ++++++++++++++++++ modoboa_installer/scripts/base.py | 45 +++- modoboa_installer/scripts/dovecot.py | 46 +++- .../files/dovecot/conf.d/10-ssl-keys.try.tpl | 6 + .../files/dovecot/conf.d/10-ssl.conf.tpl | 13 +- .../scripts/files/modoboa/crontab.tpl | 2 +- modoboa_installer/scripts/modoboa.py | 4 + modoboa_installer/scripts/nginx.py | 2 +- modoboa_installer/scripts/opendkim.py | 24 +- modoboa_installer/scripts/postfix.py | 8 +- modoboa_installer/scripts/postwhite.py | 24 +- modoboa_installer/scripts/radicale.py | 25 +- modoboa_installer/scripts/restore.py | 26 ++ modoboa_installer/utils.py | 96 +++++++- run.py | 154 ++++++++++-- 23 files changed, 863 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/installer.yml delete mode 100644 .travis.yml create mode 100644 modoboa_installer/constants.py create mode 100644 modoboa_installer/scripts/backup.py create mode 100644 modoboa_installer/scripts/files/dovecot/conf.d/10-ssl-keys.try.tpl create mode 100644 modoboa_installer/scripts/restore.py diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml new file mode 100644 index 0000000..df9bfd6 --- /dev/null +++ b/.github/workflows/installer.yml @@ -0,0 +1,60 @@ +name: Modoboa installer + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + fail-fast: false + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install -r test-requirements.txt + - name: Run tests + if: ${{ matrix.python-version != '3.9' }} + run: | + python tests.py + - name: Run tests and coverage + if: ${{ matrix.python-version == '3.9' }} + run: | + coverage run tests.py + - name: Upload coverage result + if: ${{ matrix.python-version == '3.9' }} + uses: actions/upload-artifact@v2 + with: + name: coverage-results + path: .coverage + + coverage: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + - name: Install dependencies + run: | + pip install codecov + - name: Download coverage results + uses: actions/download-artifact@v2 + with: + name: coverage-results + - name: Report coverage + run: | + coverage report + codecov diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 641df12..0000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -sudo: false -language: python -cache: pip -python: - - "2.7" - - "3.4" - -before_install: - - pip install -r test-requirements.txt - -script: - - coverage run tests.py - -after_success: - - codecov diff --git a/README.rst b/README.rst index 04da7d7..4c58cda 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ modoboa-installer ================= -|travis| |codecov| +|workflow| |codecov| An installer which deploy a complete mail server based on Modoboa. @@ -93,6 +93,54 @@ You can activate it as follows:: It will automatically install latest versions of modoboa and its plugins. +Backup mode +------------ + +An experimental backup mode is available. + +.. warning:: + + You must keep the original configuration file, i.e. the one used for + the installation. Otherwise, you will need to recreate it manually with the right information! + +You can start the process as follows:: + + $ sudo ./run.py --backup + +Then follow the step on the console. + +There is also a non-interactive mode: + +1. Silent mode + +Command:: + + $ sudo ./run.py --silent-backup + +This mode will run silently. When executed, it will create +/modoboa_backup/ and each time you execute it, it will create a new +backup directory with current date and time. + +You can supply a custom path if needed:: + + $ sudo ./run.py --silent-backup --backup-path /path/of/backup/directory + +If you want to disable emails backup, disable dovecot in the +configuration file (set enabled to False). + +This can be useful for larger instance. + +Restore mode +------------ + +An experimental restore mode is available. + +You can start the process as follows:: + + $ sudo ./run.py --restore /path/to/backup/directory/ + +Then wait for the process to finish. + Change the generated hostname ----------------------------- @@ -136,7 +184,6 @@ modify the following settings:: Change the ``email`` setting to a valid value since it will be used for account recovery. -.. |travis| image:: https://travis-ci.org/modoboa/modoboa-installer.png?branch=master - :target: https://travis-ci.org/modoboa/modoboa-installer +.. |workflow| image:: https://github.com/modoboa/modoboa-installer/workflows/Modoboa%20installer/badge.svg .. |codecov| image:: http://codecov.io/github/modoboa/modoboa-installer/coverage.svg?branch=master :target: http://codecov.io/github/modoboa/modoboa-installer?branch=master diff --git a/modoboa_installer/config_dict_template.py b/modoboa_installer/config_dict_template.py index fe04277..fea3b94 100644 --- a/modoboa_installer/config_dict_template.py +++ b/modoboa_installer/config_dict_template.py @@ -1,6 +1,8 @@ import random import string +from .constants import DEFAULT_BACKUP_DIRECTORY + def make_password(length=16): """Create a random password.""" @@ -210,7 +212,7 @@ ConfigDictTemplate = [ }, { "option": "max_servers", - "default": "1", + "default": "2", }, { "option": "dbname", @@ -439,4 +441,13 @@ ConfigDictTemplate = [ ] }, + { + "name": "backup", + "values": [ + { + "option": "default_path", + "default": DEFAULT_BACKUP_DIRECTORY + } + ] + } ] diff --git a/modoboa_installer/constants.py b/modoboa_installer/constants.py new file mode 100644 index 0000000..7df5082 --- /dev/null +++ b/modoboa_installer/constants.py @@ -0,0 +1 @@ +DEFAULT_BACKUP_DIRECTORY = "./modoboa_backup/" diff --git a/modoboa_installer/database.py b/modoboa_installer/database.py index b8a9d1c..e951869 100644 --- a/modoboa_installer/database.py +++ b/modoboa_installer/database.py @@ -134,6 +134,15 @@ class PostgreSQL(Database): self.dbhost, self.dbport, dbname, dbuser, path) utils.exec_cmd(cmd, sudo_user=self.dbuser) + def dump_database(self, dbname, dbuser, dbpassword, path): + """Dump DB to SQL file.""" + # Reset pgpass since we backup multiple db (different secret set) + self._pgpass_done = False + self._setup_pgpass(dbname, dbuser, dbpassword) + cmd = "pg_dump -h {} -d {} -U {} -O -w > {}".format( + self.dbhost, dbname, dbuser, path) + utils.exec_cmd(cmd, sudo_user=self.dbuser) + class MySQL(Database): @@ -246,6 +255,12 @@ class MySQL(Database): self.dbhost, self.dbport, dbuser, dbpassword, dbname, path) ) + def dump_database(self, dbname, dbuser, dbpassword, path): + """Dump DB to SQL file.""" + cmd = "mysqldump -h {} -u {} -p{} {} > {}".format( + self.dbhost, dbuser, dbpassword, dbname, path) + utils.exec_cmd(cmd, sudo_user=self.dbuser) + def get_backend(config): """Return appropriate backend.""" diff --git a/modoboa_installer/scripts/__init__.py b/modoboa_installer/scripts/__init__.py index 3edfa66..d006f4c 100644 --- a/modoboa_installer/scripts/__init__.py +++ b/modoboa_installer/scripts/__init__.py @@ -6,20 +6,49 @@ import sys from .. import utils -def install(appname, config, upgrade): - """Install an application.""" - if (config.has_option(appname, "enabled") and - not config.getboolean(appname, "enabled")): - return - utils.printcolor("Installing {}".format(appname), utils.MAGENTA) +def load_app_script(appname): + """Load module corresponding to the given appname.""" try: script = importlib.import_module( "modoboa_installer.scripts.{}".format(appname)) except ImportError: print("Unknown application {}".format(appname)) sys.exit(1) + return script + + +def install(appname: str, config, upgrade: bool, archive_path: str): + """Install an application.""" + if (config.has_option(appname, "enabled") and + not config.getboolean(appname, "enabled")): + return + + utils.printcolor("Installing {}".format(appname), utils.MAGENTA) + script = load_app_script(appname) try: - getattr(script, appname.capitalize())(config, upgrade).run() + getattr(script, appname.capitalize())(config, upgrade, archive_path).run() except utils.FatalError as inst: - utils.printcolor(u"{}".format(inst), utils.RED) + utils.error("{}".format(inst)) sys.exit(1) + + +def backup(appname, config, path): + """Backup an application.""" + if (config.has_option(appname, "enabled") and + not config.getboolean(appname, "enabled")): + return + + utils.printcolor("Backing up {}".format(appname), utils.MAGENTA) + script = load_app_script(appname) + try: + getattr(script, appname.capitalize())(config, False, False).backup(path) + except utils.FatalError as inst: + utils.error("{}".format(inst)) + sys.exit(1) + + +def restore_prep(restore): + """Restore instance""" + script = importlib.import_module( + "modoboa_installer.scripts.restore") + getattr(script, "Restore")(restore) diff --git a/modoboa_installer/scripts/amavis.py b/modoboa_installer/scripts/amavis.py index dabddae..01ac7d1 100644 --- a/modoboa_installer/scripts/amavis.py +++ b/modoboa_installer/scripts/amavis.py @@ -1,13 +1,12 @@ """Amavis related functions.""" import os -import platform from .. import package from .. import utils from . import base -from . import install +from . import backup, install class Amavis(base.Installer): @@ -83,7 +82,7 @@ class Amavis(base.Installer): path = self.get_file_path( "amavis_{}_{}.sql".format(self.dbengine, version)) if not os.path.exists(path): - raise utils.FatalError("Failed to find amavis database schema") + raise utils.FatalError("Failed to find amavis database schema") return path def pre_run(self): @@ -93,5 +92,25 @@ class Amavis(base.Installer): def post_run(self): """Additional tasks.""" - install("spamassassin", self.config, self.upgrade) - install("clamav", self.config, self.upgrade) + install("spamassassin", self.config, self.upgrade, self.archive_path) + install("clamav", self.config, self.upgrade, self.archive_path) + + def custom_backup(self, path): + """Backup custom configuration if any.""" + if package.backend.FORMAT == "deb": + amavis_custom = f"{self.config_dir}/conf.d/99-custom" + if os.path.isfile(amavis_custom): + utils.copy_file(amavis_custom, path) + utils.success("Amavis custom configuration saved!") + backup("spamassassin", self.config, os.path.dirname(path)) + + def restore(self): + """Restore custom config files.""" + if package.backend.FORMAT != "deb": + return + amavis_custom_configuration = os.path.join( + self.archive_path, "custom/99-custom") + if os.path.isfile(amavis_custom_configuration): + utils.copy_file(amavis_custom_configuration, os.path.join( + self.config_dir, "conf.d")) + utils.success("Custom amavis configuration restored.") diff --git a/modoboa_installer/scripts/backup.py b/modoboa_installer/scripts/backup.py new file mode 100644 index 0000000..1a6024c --- /dev/null +++ b/modoboa_installer/scripts/backup.py @@ -0,0 +1,231 @@ +"""Backup script for pre-installed instance.""" + +import os +import pwd +import shutil +import stat +import sys +import datetime + +from .. import database +from .. import utils +from ..constants import DEFAULT_BACKUP_DIRECTORY + + +class Backup: + """ + Backup structure ( {optional} ): + {{backup_directory}} + || + ||--> installer.cfg + ||--> custom + |--> { (copy of) /etc/amavis/conf.d/99-custom } + |--> { (copy of) /etc/postfix/custom_whitelist.cidr } + |--> { (copy of) dkim directory } + |--> {dkim.pem}... + |--> { (copy of) radicale home_dir } + ||--> databases + |--> modoboa.sql + |--> { amavis.sql } + |--> { spamassassin.sql } + ||--> mails + |--> vmails + """ + + def __init__(self, config, silent_backup, backup_path, nomail): + self.config = config + self.backup_path = backup_path + self.nomail = nomail + self.silent_backup = silent_backup + + def validate_path(self, path): + """Check basic condition for backup directory.""" + + path_exists = os.path.exists(path) + + if path_exists and os.path.isfile(path): + utils.printcolor( + "Error, you provided a file instead of a directory!", utils.RED) + return False + + if not path_exists: + if not self.silent_backup: + create_dir = input( + f"\"{path}\" doesn't exist, would you like to create it? [Y/n]\n").lower() + + if self.silent_backup or (not self.silent_backup and create_dir.startswith("y")): + pw = pwd.getpwnam("root") + utils.mkdir_safe(path, stat.S_IRWXU | + stat.S_IRWXG, pw[2], pw[3]) + else: + utils.printcolor( + "Error, backup directory not present.", utils.RED + ) + return False + + if len(os.listdir(path)) != 0: + if not self.silent_backup: + delete_dir = input( + "Warning: backup directory is not empty, it will be purged if you continue... [Y/n]\n").lower() + + if self.silent_backup or (not self.silent_backup and delete_dir.startswith("y")): + try: + os.remove(os.path.join(path, "installer.cfg")) + except FileNotFoundError: + pass + + shutil.rmtree(os.path.join(path, "custom"), + ignore_errors=False) + shutil.rmtree(os.path.join(path, "mails"), ignore_errors=False) + shutil.rmtree(os.path.join(path, "databases"), + ignore_errors=False) + else: + utils.printcolor( + "Error: backup directory not clean.", utils.RED + ) + return False + + self.backup_path = path + + pw = pwd.getpwnam("root") + for dir in ["custom/", "databases/"]: + utils.mkdir_safe(os.path.join(self.backup_path, dir), + stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3]) + return True + + def set_path(self): + """Setup backup directory.""" + if self.silent_backup: + if self.backup_path is None: + if self.config.has_option("backup", "default_path"): + path = self.config.get("backup", "default_path") + else: + path = DEFAULT_BACKUP_DIRECTORY + date = datetime.datetime.now().strftime("%m_%d_%Y_%H_%M") + path = os.path.join(path, f"backup_{date}") + self.validate_path(path) + else: + if not self.validate_path(self.backup_path): + utils.printcolor( + f"Path provided: {self.backup_path}", utils.BLUE) + sys.exit(1) + else: + user_value = None + while user_value == "" or user_value is None or not self.validate_path(user_value): + utils.printcolor( + "Enter backup path (it must be an empty directory)", utils.MAGENTA) + utils.printcolor("CTRL+C to cancel", utils.MAGENTA) + user_value = utils.user_input("-> ") + + def config_file_backup(self): + utils.copy_file("installer.cfg", self.backup_path) + + def mail_backup(self): + if self.nomail: + utils.printcolor( + "Skipping mail backup, no-mail argument provided", utils.MAGENTA) + return + + utils.printcolor("Backing up mails", utils.MAGENTA) + + home_path = self.config.get("dovecot", "home_dir") + + if not os.path.exists(home_path) or os.path.isfile(home_path): + utils.printcolor("Error backing up Email, provided path " + f" ({home_path}) seems not right...", utils.RED) + + else: + dst = os.path.join(self.backup_path, "mails/") + + if os.path.exists(dst): + shutil.rmtree(dst) + + shutil.copytree(home_path, dst) + utils.printcolor("Mail backup complete!", utils.GREEN) + + def custom_config_backup(self): + """ + Custom config : + - DKIM keys: {{keys_storage_dir}} + - Radicale collection (calendat, contacts): {{home_dir}} + - Amavis : /etc/amavis/conf.d/99-custom + - Postwhite : /etc/postwhite.conf + Feel free to suggest to add others! + """ + utils.printcolor( + "Backing up some custom configuration...", utils.MAGENTA) + + custom_path = os.path.join( + self.backup_path, "custom") + + # DKIM Key + if (self.config.has_option("opendkim", "enabled") and + self.config.getboolean("opendkim", "enabled")): + dkim_keys = self.config.get( + "opendkim", "keys_storage_dir", fallback="/var/lib/dkim") + if os.path.isdir(dkim_keys): + shutil.copytree(dkim_keys, os.path.join(custom_path, "dkim")) + utils.printcolor( + "DKIM keys saved!", utils.GREEN) + + # Radicale Collections + if (self.config.has_option("radicale", "enabled") and + self.config.getboolean("radicale", "enabled")): + radicale_backup = os.path.join(self.config.get( + "radicale", "home_dir", fallback="/srv/radicale"), "collections") + if os.path.isdir(radicale_backup): + shutil.copytree(radicale_backup, os.path.join( + custom_path, "radicale")) + utils.printcolor("Radicale files saved", utils.GREEN) + + # AMAVIS + if (self.config.has_option("amavis", "enabled") and + self.config.getboolean("amavis", "enabled")): + amavis_custom = "/etc/amavis/conf.d/99-custom" + if os.path.isfile(amavis_custom): + utils.copy_file(amavis_custom, custom_path) + utils.printcolor( + "Amavis custom configuration saved!", utils.GREEN) + + # POSTWHITE + if (self.config.has_option("postwhite", "enabled") and + self.config.getboolean("postwhite", "enabled")): + postswhite_custom = "/etc/postwhite.conf" + if os.path.isfile(postswhite_custom): + utils.copy_file(postswhite_custom, custom_path) + utils.printcolor( + "Postwhite configuration saved!", utils.GREEN) + + def database_backup(self): + """Backing up databases""" + + utils.printcolor("Backing up databases...", utils.MAGENTA) + + self.database_dump("modoboa") + self.database_dump("amavis") + self.database_dump("spamassassin") + + def database_dump(self, app_name): + + dump_path = os.path.join(self.backup_path, "databases") + backend = database.get_backend(self.config) + + if app_name == "modoboa" or (self.config.has_option(app_name, "enabled") and + self.config.getboolean(app_name, "enabled")): + dbname = self.config.get(app_name, "dbname") + dbuser = self.config.get(app_name, "dbuser") + dbpasswd = self.config.get(app_name, "dbpassword") + backend.dump_database(dbname, dbuser, dbpasswd, + os.path.join(dump_path, f"{app_name}.sql")) + + def backup_completed(self): + utils.printcolor("Backup process done, your backup is available here:" + f"--> {self.backup_path}", utils.GREEN) + + def run(self): + self.set_path() + self.config_file_backup() + self.mail_backup() + self.custom_config_backup() + self.database_backup() + self.backup_completed() diff --git a/modoboa_installer/scripts/base.py b/modoboa_installer/scripts/base.py index 8627132..dea0897 100644 --- a/modoboa_installer/scripts/base.py +++ b/modoboa_installer/scripts/base.py @@ -20,10 +20,11 @@ class Installer(object): with_db = False config_files = [] - def __init__(self, config, upgrade): + def __init__(self, config, upgrade: bool, archive_path: str): """Get configuration.""" self.config = config self.upgrade = upgrade + self.archive_path = archive_path if self.config.has_section(self.appname): self.app_config = dict(self.config.items(self.appname)) self.dbengine = self.config.get("database", "engine") @@ -53,6 +54,19 @@ class Installer(object): """Return a schema to install.""" return None + def get_sql_schema_from_backup(self): + """Retrieve a dump path from a previous backup.""" + utils.printcolor( + f"Trying to restore {self.appname} database from backup.", + utils.MAGENTA + ) + database_backup_path = os.path.join( + self.archive_path, f"databases/{self.appname}.sql") + if os.path.isfile(database_backup_path): + utils.success(f"SQL dump found in backup for {self.appname}!") + return database_backup_path + return None + def get_file_path(self, fname): """Return the absolute path of this file.""" return os.path.abspath( @@ -66,7 +80,11 @@ class Installer(object): return self.backend.create_user(self.dbuser, self.dbpasswd) self.backend.create_database(self.dbname, self.dbuser) - schema = self.get_sql_schema_path() + schema = None + if self.archive_path: + schema = self.get_sql_schema_from_backup() + if not schema: + schema = self.get_sql_schema_path() if schema: self.backend.load_sql_file( self.dbname, self.dbuser, self.dbpasswd, schema) @@ -137,6 +155,20 @@ class Installer(object): dst = os.path.join(self.config_dir, dst) utils.copy_from_template(src, dst, context) + def backup(self, path): + if self.with_db: + self._dump_database(path) + custom_backup_path = os.path.join(path, "custom") + self.custom_backup(custom_backup_path) + + def custom_backup(self, path): + """Override this method in subscripts to add custom backup content.""" + pass + + def restore(self): + """Restore from a previous backup.""" + pass + def get_daemon_name(self): """Return daemon name if defined.""" return self.daemon_name if self.daemon_name else self.appname @@ -157,8 +189,17 @@ class Installer(object): self.setup_database() self.install_config_files() self.post_run() + if self.archive_path: + self.restore() self.restart_daemon() + def _dump_database(self, backup_path: str): + """Create a new database dump for this app.""" + target_dir = os.path.join(backup_path, "databases") + target_file = os.path.join(target_dir, f"{self.appname}.sql") + self.backend.dump_database( + self.dbname, self.dbuser, self.dbpasswd, target_file) + def pre_run(self): """Tasks to execute before the installer starts.""" pass diff --git a/modoboa_installer/scripts/dovecot.py b/modoboa_installer/scripts/dovecot.py index 92b2bff..a1f5988 100644 --- a/modoboa_installer/scripts/dovecot.py +++ b/modoboa_installer/scripts/dovecot.py @@ -3,6 +3,7 @@ import glob import os import pwd +import shutil from .. import database from .. import package @@ -26,7 +27,7 @@ class Dovecot(base.Installer): } config_files = [ "dovecot.conf", "dovecot-dict-sql.conf.ext", "conf.d/10-ssl.conf", - "conf.d/10-master.conf", "conf.d/20-lmtp.conf"] + "conf.d/10-master.conf", "conf.d/20-lmtp.conf", "conf.d/10-ssl-keys.try"] with_user = True def get_config_files(self): @@ -58,10 +59,16 @@ class Dovecot(base.Installer): """Additional variables.""" context = super(Dovecot, self).get_template_context() pw = pwd.getpwnam(self.user) + dovecot_package = {"deb": "dovecot-core", "rpm": "dovecot"} + ssl_protocol_parameter = "ssl_protocols" + if package.backend.get_installed_version(dovecot_package[package.backend.FORMAT]) > "2.3": + ssl_protocol_parameter = "ssl_min_protocol" ssl_protocols = "!SSLv2 !SSLv3" if package.backend.get_installed_version("openssl").startswith("1.1") \ or package.backend.get_installed_version("openssl").startswith("3"): ssl_protocols = "!SSLv3" + if ssl_protocol_parameter == "ssl_min_protocol": + ssl_protocols = "TLSv1" if "centos" in utils.dist_name(): protocols = "protocols = imap lmtp sieve" extra_protocols = self.config.get("dovecot", "extra_protocols") @@ -80,6 +87,7 @@ class Dovecot(base.Installer): "modoboa_dbpassword": self.config.get("modoboa", "dbpassword"), "protocols": protocols, "ssl_protocols": ssl_protocols, + "ssl_protocol_parameter": ssl_protocol_parameter, "radicale_user": self.config.get("radicale", "user"), "radicale_auth_socket_path": os.path.basename( self.config.get("dovecot", "radicale_auth_socket_path")) @@ -126,3 +134,39 @@ class Dovecot(base.Installer): "service {} {} > /dev/null 2>&1".format(self.appname, action), capture_output=False) system.enable_service(self.get_daemon_name()) + + def backup(self, path): + """Backup emails.""" + home_dir = self.config.get("dovecot", "home_dir") + utils.printcolor("Backing up mails", utils.MAGENTA) + if not os.path.exists(home_dir) or os.path.isfile(home_dir): + utils.error("Error backing up emails, provided path " + f" ({home_dir}) seems not right...") + return + + dst = os.path.join(path, "mails/") + if os.path.exists(dst): + shutil.rmtree(dst) + shutil.copytree(home_dir, dst) + utils.success("Mail backup complete!") + + def restore(self): + """Restore emails.""" + home_dir = self.config.get("dovecot", "home_dir") + mail_dir = os.path.join(self.archive_path, "mails/") + if len(os.listdir(mail_dir)) > 0: + utils.success("Copying mail backup over dovecot directory.") + if os.path.exists(home_dir): + shutil.rmtree(home_dir) + shutil.copytree(mail_dir, home_dir) + # Resetting permission for vmail + for dirpath, dirnames, filenames in os.walk(home_dir): + shutil.chown(dirpath, self.user, self.user) + for filename in filenames: + shutil.chown(os.path.join(dirpath, filename), + self.user, self.user) + else: + utils.printcolor( + "It seems that emails were not backed up, skipping restoration.", + utils.MAGENTA + ) diff --git a/modoboa_installer/scripts/files/dovecot/conf.d/10-ssl-keys.try.tpl b/modoboa_installer/scripts/files/dovecot/conf.d/10-ssl-keys.try.tpl new file mode 100644 index 0000000..e44abb8 --- /dev/null +++ b/modoboa_installer/scripts/files/dovecot/conf.d/10-ssl-keys.try.tpl @@ -0,0 +1,6 @@ +# PEM encoded X.509 SSL/TLS certificate and private key. They're opened before +# dropping root privileges, so keep the key file unreadable by anyone but +# root. Included doc/mkcert.sh can be used to easily generate self-signed +# certificate, just make sure to update the domains in dovecot-openssl.cnf +ssl_cert = <%tls_cert_file +ssl_key = <%tls_key_file \ No newline at end of file diff --git a/modoboa_installer/scripts/files/dovecot/conf.d/10-ssl.conf.tpl b/modoboa_installer/scripts/files/dovecot/conf.d/10-ssl.conf.tpl index 80400a6..35382ca 100644 --- a/modoboa_installer/scripts/files/dovecot/conf.d/10-ssl.conf.tpl +++ b/modoboa_installer/scripts/files/dovecot/conf.d/10-ssl.conf.tpl @@ -5,12 +5,11 @@ # SSL/TLS support: yes, no, required. #ssl = yes -# PEM encoded X.509 SSL/TLS certificate and private key. They're opened before -# dropping root privileges, so keep the key file unreadable by anyone but -# root. Included doc/mkcert.sh can be used to easily generate self-signed -# certificate, just make sure to update the domains in dovecot-openssl.cnf -ssl_cert = <%tls_cert_file -ssl_key = <%tls_key_file +# Workarround https://github.com/modoboa/modoboa/issues/2570 +# We try to load the key and pass if it fails +# Keys require root permissions, standard commands would be blocked +# because dovecot can't load these cert +!include_try /etc/dovecot/conf.d/10-ssl-keys.try # If key file is password protected, give the password here. Alternatively # give it when starting dovecot with -p parameter. Since this file is often @@ -41,7 +40,7 @@ ssl_key = <%tls_key_file #ssl_parameters_regenerate = 168 # SSL protocols to use -ssl_min_protocol = %ssl_protocols +%ssl_protocol_parameter = %ssl_protocols # SSL ciphers to use diff --git a/modoboa_installer/scripts/files/modoboa/crontab.tpl b/modoboa_installer/scripts/files/modoboa/crontab.tpl index a50b2df..28bde36 100644 --- a/modoboa_installer/scripts/files/modoboa/crontab.tpl +++ b/modoboa_installer/scripts/files/modoboa/crontab.tpl @@ -30,7 +30,7 @@ INSTANCE=%{instance_path} */30 * * * * root $PYTHON $INSTANCE/manage.py modo check_mx # Public API communication -0 * * * * root $PYTHON $INSTANCE/manage.py communicate_with_public_api +%{minutes} %{hours} * * * root $PYTHON $INSTANCE/manage.py communicate_with_public_api # Generate DKIM keys (they will belong to the user running this job) %{opendkim_enabled}* * * * * %{opendkim_user} umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys diff --git a/modoboa_installer/scripts/modoboa.py b/modoboa_installer/scripts/modoboa.py index ea1320c..be99db0 100644 --- a/modoboa_installer/scripts/modoboa.py +++ b/modoboa_installer/scripts/modoboa.py @@ -3,6 +3,7 @@ import json import os import pwd +import random import shutil import stat import sys @@ -219,6 +220,7 @@ class Modoboa(base.Installer): context = super(Modoboa, self).get_template_context() extensions = self.config.get("modoboa", "extensions") extensions = extensions.split() + random_hour = random.randint(0, 6) context.update({ "sudo_user": ( "uwsgi" if package.backend.FORMAT == "rpm" else context["user"] @@ -228,6 +230,8 @@ class Modoboa(base.Installer): "radicale_enabled": ( "" if "modoboa-radicale" in extensions else "#"), "opendkim_user": self.config.get("opendkim", "user"), + "minutes": random.randint(1, 59), + "hours" : f"{random_hour},{random_hour+12}" }) return context diff --git a/modoboa_installer/scripts/nginx.py b/modoboa_installer/scripts/nginx.py index bfa0850..3537842 100644 --- a/modoboa_installer/scripts/nginx.py +++ b/modoboa_installer/scripts/nginx.py @@ -26,7 +26,7 @@ class Nginx(base.Installer): "app_instance_path": ( self.config.get(app, "instance_path")), "uwsgi_socket_path": ( - Uwsgi(self.config, self.upgrade).get_socket_path(app)) + Uwsgi(self.config, self.upgrade, self.restore).get_socket_path(app)) }) return context diff --git a/modoboa_installer/scripts/opendkim.py b/modoboa_installer/scripts/opendkim.py index 9875a93..0379221 100644 --- a/modoboa_installer/scripts/opendkim.py +++ b/modoboa_installer/scripts/opendkim.py @@ -2,6 +2,7 @@ import os import pwd +import shutil import stat from .. import database @@ -46,7 +47,7 @@ class Opendkim(base.Installer): stat.S_IROTH | stat.S_IXOTH, target[1], target[2] ) - super(Opendkim, self).install_config_files() + super().install_config_files() def get_template_context(self): """Additional variables.""" @@ -109,3 +110,24 @@ class Opendkim(base.Installer): "s/^After=(.*)$/After=$1 {}/".format(dbservice)) utils.exec_cmd( "perl -pi -e '{}' /lib/systemd/system/opendkim.service".format(pattern)) + + def restore(self): + """Restore keys.""" + dkim_keys_backup = os.path.join( + self.archive_path, "custom/dkim") + if os.path.isdir(dkim_keys_backup): + for file in os.listdir(dkim_keys_backup): + file_path = os.path.join(dkim_keys_backup, file) + if os.path.isfile(file_path): + utils.copy_file(file_path, self.config.get( + "opendkim", "keys_storage_dir", fallback="/var/lib/dkim")) + utils.success("DKIM keys restored from backup") + + def custom_backup(self, path): + """Backup DKIM keys.""" + storage_dir = self.config.get( + "opendkim", "keys_storage_dir", fallback="/var/lib/dkim") + if os.path.isdir(storage_dir): + shutil.copytree(storage_dir, os.path.join(path, "dkim")) + utils.printcolor( + "DKIM keys saved!", utils.GREEN) diff --git a/modoboa_installer/scripts/postfix.py b/modoboa_installer/scripts/postfix.py index 709a035..6601fe5 100644 --- a/modoboa_installer/scripts/postfix.py +++ b/modoboa_installer/scripts/postfix.py @@ -10,7 +10,7 @@ from .. import package from .. import utils from . import base -from . import install +from . import backup, install class Postfix(base.Installer): @@ -86,4 +86,8 @@ class Postfix(base.Installer): utils.exec_cmd("postalias {}".format(aliases_file)) # Postwhite - install("postwhite", self.config, self.upgrade) + install("postwhite", self.config, self.upgrade, self.archive_path) + + def backup(self, path): + """Launch postwhite backup.""" + backup("postwhite", self.config, path) diff --git a/modoboa_installer/scripts/postwhite.py b/modoboa_installer/scripts/postwhite.py index 039bd11..30bcb14 100644 --- a/modoboa_installer/scripts/postwhite.py +++ b/modoboa_installer/scripts/postwhite.py @@ -45,8 +45,26 @@ class Postwhite(base.Installer): """Additionnal tasks.""" install_dir = "/usr/local/bin" self.install_from_archive(SPF_TOOLS_REPOSITORY, install_dir) - postw_dir = self.install_from_archive( + self.postw_dir = self.install_from_archive( POSTWHITE_REPOSITORY, install_dir) - utils.copy_file(os.path.join(postw_dir, "postwhite.conf"), "/etc") - postw_bin = os.path.join(postw_dir, "postwhite") + postw_bin = os.path.join(self.postw_dir, "postwhite") utils.exec_cmd("{} /etc/postwhite.conf".format(postw_bin)) + + def custom_backup(self, path): + """Backup custom configuration if any.""" + postswhite_custom = "/etc/postwhite.conf" + if os.path.isfile(postswhite_custom): + utils.copy_file(postswhite_custom, path) + utils.printcolor( + "Postwhite configuration saved!", utils.GREEN) + + def restore(self): + """Restore config files.""" + postwhite_backup_configuration = os.path.join( + self.archive_path, "custom/postwhite.conf") + if os.path.isfile(postwhite_backup_configuration): + utils.copy_file(postwhite_backup_configuration, self.config_dir) + utils.success("postwhite.conf restored from backup") + else: + utils.copy_file( + os.path.join(self.postw_dir, "postwhite.conf"), self.config_dir) diff --git a/modoboa_installer/scripts/radicale.py b/modoboa_installer/scripts/radicale.py index 889403f..a0ca309 100644 --- a/modoboa_installer/scripts/radicale.py +++ b/modoboa_installer/scripts/radicale.py @@ -1,6 +1,7 @@ """Radicale related tasks.""" import os +import shutil import stat from .. import package @@ -25,7 +26,7 @@ class Radicale(base.Installer): def __init__(self, *args, **kwargs): """Get configuration.""" - super(Radicale, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.venv_path = self.config.get("radicale", "venv_path") def _setup_venv(self): @@ -70,7 +71,18 @@ class Radicale(base.Installer): stat.S_IROTH | stat.S_IXOTH, 0, 0 ) - super(Radicale, self).install_config_files() + super().install_config_files() + + def restore(self): + """Restore collections.""" + radicale_backup = os.path.join( + self.archive_path, "custom/radicale") + if os.path.isdir(radicale_backup): + restore_target = os.path.join(self.home_dir, "collections") + if os.path.isdir(restore_target): + shutil.rmtree(restore_target) + shutil.copytree(radicale_backup, restore_target) + utils.success("Radicale collections restored from backup") def post_run(self): """Additional tasks.""" @@ -81,3 +93,12 @@ class Radicale(base.Installer): system.enable_service(daemon_name) utils.exec_cmd("service {} stop".format(daemon_name)) utils.exec_cmd("service {} start".format(daemon_name)) + + def custom_backup(self, path): + """Backup collections.""" + radicale_backup = os.path.join(self.config.get( + "radicale", "home_dir", fallback="/srv/radicale"), "collections") + if os.path.isdir(radicale_backup): + shutil.copytree(radicale_backup, os.path.join( + path, "radicale")) + utils.printcolor("Radicale files saved", utils.GREEN) diff --git a/modoboa_installer/scripts/restore.py b/modoboa_installer/scripts/restore.py new file mode 100644 index 0000000..08e3abc --- /dev/null +++ b/modoboa_installer/scripts/restore.py @@ -0,0 +1,26 @@ +import os +import sys +from .. import utils + + +class Restore: + def __init__(self, restore): + """ + Restoring pre-check (backup integriety) + REQUIRED : modoboa.sql + OPTIONAL : mails/, custom/, amavis.sql, spamassassin.sql + Only checking required + """ + + if not os.path.isdir(restore): + utils.printcolor( + "Provided path is not a directory !", utils.RED) + sys.exit(1) + + modoba_sql_file = os.path.join(restore, "databases/modoboa.sql") + if not os.path.isfile(modoba_sql_file): + utils.printcolor( + modoba_sql_file + " not found, please check your backup", utils.RED) + sys.exit(1) + + # Everything seems alright here, proceeding... diff --git a/modoboa_installer/utils.py b/modoboa_installer/utils.py index 8a793af..5f97ccb 100644 --- a/modoboa_installer/utils.py +++ b/modoboa_installer/utils.py @@ -2,10 +2,13 @@ import contextlib import datetime +import getpass import glob import os +import pwd import random import shutil +import stat import string import subprocess import sys @@ -107,6 +110,13 @@ def mkdir(path, mode, uid, gid): os.chown(path, uid, gid) +def mkdir_safe(path, mode, uid, gid): + """Create a directory. Safe way (-p)""" + if not os.path.exists(path): + os.makedirs(os.path.abspath(path), mode) + mkdir(path, mode, uid, gid) + + def make_password(length=16): """Create a random password.""" return "".join( @@ -163,19 +173,32 @@ def copy_from_template(template, dest, context): fp.write(ConfigFileTemplate(buf).substitute(context)) -def check_config_file(dest, interactive=False, upgrade=False): +def check_config_file(dest, interactive=False, upgrade=False, backup=False, restore=False): """Create a new installer config file if needed.""" + is_present = True if os.path.exists(dest): - return + return is_present if upgrade: printcolor( "You cannot upgrade an existing installation without a " "configuration file.", RED) sys.exit(1) + elif backup: + is_present = False + printcolor( + "Your configuration file hasn't been found. A new one will be generated. " + "Please edit it with correct password for the databases !", RED) + elif restore: + printcolor( + "You cannot restore an existing installation without a " + f"configuration file. (file : {dest} has not been found...", RED) + sys.exit(1) + printcolor( "Configuration file {} not found, creating new one." .format(dest), YELLOW) gen_config(dest, interactive) + return is_present def has_colours(stream): @@ -203,6 +226,16 @@ def printcolor(message, color): print(message) +def error(message): + """Print error message.""" + printcolor(message, RED) + + +def success(message): + """Print success message.""" + printcolor(message, GREEN) + + def convert_version_to_int(version): """Convert a version string to an integer.""" number_bits = (8, 8, 16) @@ -307,3 +340,62 @@ def gen_config(dest, interactive=False): with open(dest, "w") as configfile: config.write(configfile) + + # Set file owner to running user and group, and set config file permission to 600 + current_username = getpass.getuser() + current_user = pwd.getpwnam(current_username) + os.chown(dest, current_user[2], current_user[3]) + os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR) + + +def validate_backup_path(path: str, silent_mode: bool): + """Check if provided backup path is valid or not.""" + path_exists = os.path.exists(path) + if path_exists and os.path.isfile(path): + printcolor( + "Error, you provided a file instead of a directory!", RED) + return None + + if not path_exists: + if not silent_mode: + create_dir = input( + f"\"{path}\" doesn't exist, would you like to create it? [Y/n]\n" + ).lower() + + if silent_mode or (not silent_mode and create_dir.startswith("y")): + pw = pwd.getpwnam("root") + mkdir_safe(path, stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3]) + else: + printcolor( + "Error, backup directory not present.", RED + ) + return None + + if len(os.listdir(path)) != 0: + if not silent_mode: + delete_dir = input( + "Warning: backup directory is not empty, it will be purged if you continue... [Y/n]\n").lower() + + if silent_mode or (not silent_mode and delete_dir.startswith("y")): + try: + os.remove(os.path.join(path, "installer.cfg")) + except FileNotFoundError: + pass + + shutil.rmtree(os.path.join(path, "custom"), + ignore_errors=False) + shutil.rmtree(os.path.join(path, "mails"), ignore_errors=False) + shutil.rmtree(os.path.join(path, "databases"), + ignore_errors=False) + else: + printcolor( + "Error: backup directory not clean.", RED + ) + return None + + backup_path = path + pw = pwd.getpwnam("root") + for dir in ["custom/", "databases/"]: + mkdir_safe(os.path.join(backup_path, dir), + stat.S_IRWXU | stat.S_IRWXG, pw[2], pw[3]) + return backup_path diff --git a/run.py b/run.py index 8bfeaf2..88c7ed6 100755 --- a/run.py +++ b/run.py @@ -3,6 +3,8 @@ """An installer for Modoboa.""" import argparse +import datetime +import os try: import configparser except ImportError: @@ -10,6 +12,7 @@ except ImportError: import sys from modoboa_installer import compatibility_matrix +from modoboa_installer import constants from modoboa_installer import package from modoboa_installer import scripts from modoboa_installer import ssl @@ -17,6 +20,19 @@ from modoboa_installer import system from modoboa_installer import utils +PRIMARY_APPS = [ + "amavis", + "modoboa", + "automx", + "radicale", + "uwsgi", + "nginx", + "opendkim", + "postfix", + "dovecot" +] + + def installation_disclaimer(args, config): """Display installation disclaimer.""" hostname = config.get("general", "hostname") @@ -45,6 +61,60 @@ def upgrade_disclaimer(config): ) +def backup_disclaimer(): + """Display backup disclamer. """ + utils.printcolor( + "Your mail server will be backed up locally.\n" + " !! You should really transfer the backup somewhere else...\n" + " !! Custom configuration (like for postfix) won't be saved.", utils.BLUE) + + +def restore_disclaimer(): + """Display restore disclamer. """ + utils.printcolor( + "You are about to restore a previous installation of Modoboa.\n" + "If a new version has been released in between, please update your database!", + utils.BLUE) + + +def backup_system(config, args): + """Launch backup procedure.""" + backup_disclaimer() + backup_path = None + if args.silent_backup: + if not args.backup_path: + if config.has_option("backup", "default_path"): + path = config.get("backup", "default_path") + else: + path = constants.DEFAULT_BACKUP_DIRECTORY + date = datetime.datetime.now().strftime("%m_%d_%Y_%H_%M") + path = os.path.join(path, f"backup_{date}") + else: + path = args.backup_path + backup_path = utils.validate_backup_path(path, args.silent_backup) + if not backup_path: + utils.printcolor(f"Path provided: {path}", utils.BLUE) + return + else: + user_value = None + while not user_value or not backup_path: + utils.printcolor( + "Enter backup path (it must be an empty directory)", + utils.MAGENTA + ) + utils.printcolor("CTRL+C to cancel", utils.MAGENTA) + user_value = utils.user_input("-> ") + if not user_value: + continue + backup_path = utils.validate_backup_path(user_value, args.silent_backup) + + # Backup configuration file + utils.copy_file(args.configfile, backup_path) + # Backup applications + for app in PRIMARY_APPS: + scripts.backup(app, config, backup_path) + + def main(input_args): """Install process.""" parser = argparse.ArgumentParser() @@ -72,16 +142,52 @@ def main(input_args): parser.add_argument( "--beta", action="store_true", default=False, help="Install latest beta release of Modoboa instead of the stable one") + parser.add_argument( + "--backup-path", type=str, metavar="path", + help="To use with --silent-backup, you must provide a valid path") + parser.add_argument( + "--backup", action="store_true", default=False, + help="Backing up interactively previously installed instance" + ) + parser.add_argument( + "--silent-backup", action="store_true", default=False, + help="For script usage, do not require user interaction " + "backup will be saved at ./modoboa_backup/Backup_M_Y_d_H_M if --backup-path is not provided") + parser.add_argument( + "--restore", type=str, metavar="path", + help="Restore a previously backup up modoboa instance on a NEW machine. " + "You MUST provide backup directory" + ) parser.add_argument("domain", type=str, help="The main domain of your future mail server") args = parser.parse_args(input_args) if args.debug: utils.ENV["debug"] = True - utils.printcolor("Welcome to Modoboa installer!\n", utils.GREEN) - utils.check_config_file(args.configfile, args.interactive, args.upgrade) + + # Restore prep + is_restoring = False + if args.restore is not None: + is_restoring = True + args.configfile = os.path.join(args.restore, args.configfile) + if not os.path.exists(args.configfile): + utils.error( + "Installer configuration file not found in backup!" + ) + sys.exit(1) + + utils.success("Welcome to Modoboa installer!\n") + is_config_file_available = utils.check_config_file( + args.configfile, args.interactive, args.upgrade, args.backup, is_restoring) + + if not is_config_file_available and ( + args.upgrade or args.backup or args.silent_backup): + utils.error("No config file found.") + return + if args.stop_after_configfile_check: return + config = configparser.ConfigParser() with open(args.configfile) as fp: config.read_file(fp) @@ -91,11 +197,20 @@ def main(input_args): config.set("dovecot", "domain", args.domain) config.set("modoboa", "version", args.version) config.set("modoboa", "install_beta", str(args.beta)) - # Display disclaimerpython 3 linux distribution - if not args.upgrade: - installation_disclaimer(args, config) - else: + + if args.backup or args.silent_backup: + backup_system(config, args) + return + + # Display disclaimer python 3 linux distribution + if args.upgrade: upgrade_disclaimer(config) + elif args.restore: + restore_disclaimer() + scripts.restore_prep(args.restore) + else: + installation_disclaimer(args, config) + # Show concerned components components = [] for section in config.sections(): @@ -115,27 +230,26 @@ def main(input_args): utils.printcolor( "The process can be long, feel free to take a coffee " "and come back later ;)", utils.BLUE) - utils.printcolor("Starting...", utils.GREEN) + utils.success("Starting...") package.backend.prepare_system() package.backend.install_many(["sudo", "wget"]) ssl_backend = ssl.get_backend(config) if ssl_backend and not args.upgrade: ssl_backend.generate_cert() - scripts.install("amavis", config, args.upgrade) - scripts.install("modoboa", config, args.upgrade) - scripts.install("automx", config, args.upgrade) - scripts.install("radicale", config, args.upgrade) - scripts.install("uwsgi", config, args.upgrade) - scripts.install("nginx", config, args.upgrade) - scripts.install("opendkim", config, args.upgrade) - scripts.install("postfix", config, args.upgrade) - scripts.install("dovecot", config, args.upgrade) + for appname in PRIMARY_APPS: + scripts.install(appname, config, args.upgrade, args.restore) system.restart_service("cron") package.backend.restore_system() - utils.printcolor( - "Congratulations! You can enjoy Modoboa at https://{} (admin:password)" - .format(config.get("general", "hostname")), - utils.GREEN) + if not args.restore: + utils.success( + "Congratulations! You can enjoy Modoboa at https://{} (admin:password)" + .format(config.get("general", "hostname")) + ) + else: + utils.success( + "Restore complete! You can enjoy Modoboa at https://{} (same credentials as before)" + .format(config.get("general", "hostname")) + ) if __name__ == "__main__":