From 252c827d92339cba886a0f882b29918621196101 Mon Sep 17 00:00:00 2001 From: Viyurz Date: Tue, 8 Oct 2024 20:19:36 +0200 Subject: [PATCH] [manage.py] Add vaultwarden + complete backup support --- env.yml | 1 - manage.py | 92 ++++++++++++++++++++------ projects/vaultwarden/.env.mako | 13 ++++ projects/vaultwarden/compose.yaml.mako | 12 ++++ pyenv.yml | 7 ++ 5 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 projects/vaultwarden/.env.mako create mode 100644 projects/vaultwarden/compose.yaml.mako diff --git a/env.yml b/env.yml index 1f4b063..0e42f48 100644 --- a/env.yml +++ b/env.yml @@ -72,7 +72,6 @@ projects_to_backup: - stump - synapse - uptime-kuma - - vaultwarden borg_repodir: "{{ cifs_mounts['backups']['path'] }}/borg" diff --git a/manage.py b/manage.py index c98dfa9..c04f730 100755 --- a/manage.py +++ b/manage.py @@ -10,29 +10,73 @@ import yaml def backupProj(project): - print(f"Running backup for project {project}.") - # loop env.volumes & secrets.postgres + database = None + paths = [] + + if project in secrets['postgres'].keys(): + database = project + + if project in env['backup'].keys(): + for path in env['backup'][project]: + paths += [path] + + if database is not None or len(paths) > 0: + print(f"Running backup for project {project}.") + + if len(paths) == 0: + borgCreate(project, database=database) + else: + borgCreate(project, path=paths[0], database=database) + + for path in paths[1:]: + archiveName = re.sub('/', '-', path) + borgCreate(f"{project}-{archiveName}", path=path) def borgCreate(name, path=None, database=None): if path is None and database is None: - print(f"Backup failed, you must pass at least one parameter amongst: path, database (archive name = {name}).", file=sys.stderr) - return + print(f"Cannot create backup, you must pass at least one parameter amongst: path, database (archive name = {name}).", file=sys.stderr) + return 1 + + borgPaths = [] if path is not None: absPath = Path(path).absolute() - print(f"Creating archive '{name}' from path {absPath}.") - parentDir = absPath.parent.absolute() os.chdir(parentDir) - runBorg(["create", "--compression=lzma", f"{env['borg_repo']}::{name}-" + '{now:%Y-%m-%d_%H-%M-%S}', absPath.relative_to(parentDir)]) + borgPaths += [absPath.relative_to(parentDir)] - runBorg(["prune", "--glob-archives", f"{name}-*"] + env['borg_prune_opts'] + [env['borg_repo']]) + borgInput = None + + if database is not None: + print(f"Dumping database {database}.") + + dumpProc = subprocess.run(["docker", "exec", "postgres", "pg_dump", "-c", database], capture_output=True, text=True) + if dumpProc.returncode != 0: + print(f"Failed to dump database {database}.", file=sys.stderr) + return 1 + + borgPaths += ["-", "--stdin-name", f"{database}.sql"] + + borgInput = dumpProc.stdout + + if path is None: + print(f"Creating archive '{name}' from database {database}.") + elif database is None: + print(f"Creating archive '{name}' from path {path}.") + else: + print(f"Creating archive '{name}' from database {database} and path {absPath}.") + + borgArgs = ["create", "--compression=lzma", f"{env['borg_repo']}::{name}_{{utcnow}}"] + runBorg(borgArgs + borgPaths, input=borgInput) os.chdir(os.path.realpath(sys.path[0])) + print(f"Pruning archives '{name}_*'.") + runBorg(["prune", "--glob-archives", f"{name}_*"] + env['borg_prune_opts'] + [env['borg_repo']]) + def getImageId (image): return runPodman("image", ["inspect", "--format", "{{.Id}}", image]).stdout.strip() @@ -77,20 +121,18 @@ def renderFile(templateFile): outputFile.close() -def runBorg(args): +def runBorg(args, input=None): if isinstance(args, str): args = args.split(' ') - env = {"BORG_PASSPHRASE": secrets['borg']} - child = subprocess.Popen(["borg"] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) + borgEnv = {"BORG_PASSPHRASE": secrets['borg']} + proc = subprocess.run(["borg"] + args, input=input, capture_output=True, text=True, env=borgEnv) - child.wait() - - stderr = child.stderr.read().strip() + stderr = proc.stderr.strip() if stderr != '': print(stderr, file=sys.stderr) - return child + return proc def runPodman(cmd, args): @@ -103,9 +145,16 @@ def runPodman(cmd, args): runArgs = ["podman", cmd] + args if env['rootless']: - return subprocess.run(runArgs, capture_output=True, text=True) + proc = subprocess.run(runArgs, capture_output=True, text=True) + stderr = proc.stderr.strip() else: - return runSudo(runArgs) + proc = runSudo(runArgs) + stderr = proc.stderr.read().strip() + + if stderr != '': + print(stderr, file=sys.stderr) + + return proc def runSudo(args): @@ -184,7 +233,8 @@ def setPerms(path, mode): def setupProj(project): print(f"Running setup for project {project}.") - backupProj(project) + if os.path.isfile(f"projects/{project}/compose.yaml.rendered"): + backupProj(project) if project == 'diun': if not os.path.isfile(env['socket']): @@ -206,14 +256,16 @@ def setupProj(project): if project in env['volumes']: for volume in env['volumes'][project].values(): + if not os.path.exists(volume): + os.makedirs(volume) setPerms(volume, 750) - setOwner(volume, getUid(project), getUid(project)) + setOwner(volume, getUid(project), env['podman_uid']) upProj(project) def upProj(project): - if runPodman("container", ["exists", project]).returncode == 0: + if runPodman("pod", ["exists", f"pod_{project}"]).returncode == 0: print(f"Tearing down stack for project {project}.") runPodman("compose", ["-f", f"projects/{project}/compose.yaml.rendered", "down"]) diff --git a/projects/vaultwarden/.env.mako b/projects/vaultwarden/.env.mako new file mode 100644 index 0000000..9dd6636 --- /dev/null +++ b/projects/vaultwarden/.env.mako @@ -0,0 +1,13 @@ +ADMIN_TOKEN='${secrets["vw_admin_token_hash"]}' +DOMAIN=https://vw.${env['domain']} +ROCKET_PORT=8080 +SIGNUPS_ALLOWED=false + +DATABASE_URL=postgresql://${secrets['postgres']['vaultwarden']['user']}:${secrets['postgres']['vaultwarden']['pass']}@postgres.${env['domain']}:${env['ports']['postgres']}/vaultwarden + +SMTP_HOST=mail.${env['domain']} +SMTP_FROM=vaultwarden@${env['domain']} +SMTP_PORT=${env['ports']['mailserver_smtps']} +SMTP_SECURITY=force_tls +SMTP_USERNAME='${secrets["mailserver"]["vaultwarden"]["user"]}' +SMTP_PASSWORD='${secrets["mailserver"]["vaultwarden"]["pass"]}' diff --git a/projects/vaultwarden/compose.yaml.mako b/projects/vaultwarden/compose.yaml.mako new file mode 100644 index 0000000..43f7089 --- /dev/null +++ b/projects/vaultwarden/compose.yaml.mako @@ -0,0 +1,12 @@ +services: + vaultwarden: + container_name: vaultwarden + image: docker.io/vaultwarden/server:alpine + network_mode: pasta:-a,${env['pasta']['vaultwarden']['ipv4']},-a,${env['pasta']['vaultwarden']['ipv6']} + restart: always + user: ${env['users']['vaultwarden']}:${env['users']['vaultwarden']} + env_file: .env.rendered + ports: + - 127.0.0.1:${env['ports']['vaultwarden']}:8080 + volumes: + - ${env['volumes']['vaultwarden']['datadir']}:/data diff --git a/pyenv.yml b/pyenv.yml index c5c0fa0..f646049 100644 --- a/pyenv.yml +++ b/pyenv.yml @@ -20,6 +20,10 @@ socket: "/run/podman/podman.sock" % endif +backup: + vaultwarden: + - /mnt/vwdata/attachments + borg_repo: /mnt/storagebox/backups/borg2 borg_prune_opts: - "--keep-within=1d" @@ -48,6 +52,9 @@ pasta: syncthing_relaysrv: ipv4: 10.86.21.1 ipv6: fc86::21 + vaultwarden: + ipv4: 10.86.23.1 + ipv6: fc86::23 # Ports exposed to host