#!/usr/bin/python3 import os, sys, re import filecmp from glob import glob from mako.template import Template from pathlib import Path import subprocess import yaml def backupProj(project): database = None paths = [] sqlite_db = None if project == 'postgres': database = 'all' elif project in secrets['postgres'].keys(): database = project if project in env['backup'].keys(): for path in env['backup'][project]: paths += [path] if project in env['backup_sqlite'].keys(): sqlite_db = env['backup_sqlite'][project] if database is not None or sqlite_db is not None or len(paths) > 0: print(f"Running backup for project {project}.") if len(paths) == 0: borgCreate(project, database=database, sqlite_db=sqlite_db) else: borgCreate(project, path=paths[0], database=database, sqlite_db=sqlite_db) for path in paths[1:]: archiveName = re.sub('/', '-', path) borgCreate(f"{project}-{archiveName}", path=path) def borgCreate(name, path=None, database=None, sqlite_db=None): if path is None and database is None and sqlite_db is None: print(f"Cannot create backup, you must pass at least one parameter amongst: path, database, database_path (archive name = {name}).", file=sys.stderr) return 1 borgPaths = [] if path is not None: absPath = Path(path).absolute() parentDir = absPath.parent.absolute() os.chdir(parentDir) borgPaths += [absPath.relative_to(parentDir)] borgInput = None if database is not None: print(f"Dumping database {database}.") if database == 'all': dumpProc = subprocess.run(["podman", "exec", "postgres", "pg_dumpall"], capture_output=True, text=True) else: dumpProc = subprocess.run(["podman", "exec", "postgres", "pg_dump", "-c", database], capture_output=True, text=True) if dumpProc.returncode != 0: print(f"Failed to dump database {database}.", file=sys.stderr) return 1 borgPaths += ["-", "--stdin-name", f"{database}.sql"] borgInput = dumpProc.stdout if sqlite_db is not None: print(f"Dumping SQLite database {sqlite_db}.") sqlite_args = ["podman", "run", "--rm", "-v", f"{sqlite_db}:/db.sqlite", "docker.io/library/alpine:latest", "sh", "-c", "apk add sqlite &>/dev/null; sqlite3 /db.sqlite .dump"] dumpProc = subprocess.run(sqlite_args, capture_output=True, text=True) if dumpProc.returncode != 0: print(f"Failed to dump SQLite database {sqlite_db}.", file=sys.stderr) return 1 database = Path(sqlite_db).stem borgPaths += ["-", "--stdin-name", f"{database}.sqlite"] borgInput = dumpProc.stdout if path is None: print(f"Creating archive '{name}' from database {database}.") elif database is None: print(f"Creating archive '{name}' from path {path}.") else: print(f"Creating archive '{name}' from database {database} and path {absPath}.") borgArgs = ["create", "--compression=lzma", f"{env['borg_repo']}::{name}_{{utcnow}}"] runBorg(borgArgs + borgPaths, input=borgInput) os.chdir(os.path.realpath(sys.path[0])) print(f"Pruning archives '{name}_*'.") runBorg(["prune", "--glob-archives", f"{name}_*"] + env['borg_prune_opts'] + [env['borg_repo']]) def getImageId (image): return runPodman("image", ["inspect", "--format", "{{.Id}}", image]).stdout.strip() def getUid(service): if service in env['users'].keys() and env['users'][service] != 0: return env['users'][service] + env['uid_shift'] else: return env['podman_uid'] def pullProj(project): print(f"Pulling images for project {project}.") with open(f"projects/{project}/compose.yaml.rendered", 'r') as composefile: images = re.findall('(?<=image:\\s).+', composefile.read()) pulledImages = [] for image in images: currentId = getImageId(image) if re.search('^localhost/', image): runPodman("compose", ["-f", f"projects/{project}/compose.yaml.rendered", "build", "--pull"]) else: runPodman("pull", image) pulledId = getImageId(image) if currentId != pulledId: pulledImages += image print(f"Pulled new version of image {image}.") else: print(f"No update available for image {image}.") return pulledImages def renderFile(templateFile): print(f"Rendering file {templateFile}.") renderedFilename = re.sub('\\.mako$', '.rendered', templateFile) template = Template(filename=templateFile) outputFile = open(renderedFilename, "w") outputFile.write(template.render(env=env, secrets=secrets)) outputFile.close() def runBorg(args, input=None): if isinstance(args, str): args = args.split(' ') borgEnv = {"BORG_PASSPHRASE": secrets['borg']} proc = subprocess.run(["borg"] + args, input=input, capture_output=True, text=True, env=borgEnv) stderr = proc.stderr.strip() if stderr != '': print(stderr, file=sys.stderr) return proc def runPodman(cmd, args): if isinstance(args, str): args = args.split(' ') if cmd == 'compose': runArgs = ["podman-compose"] + args else: runArgs = ["podman", cmd] + args if env['rootless']: proc = subprocess.run(runArgs, capture_output=True, text=True) stderr = proc.stderr.strip() else: proc = runSudo(runArgs) stderr = proc.stderr.read().strip() if stderr != '': print(stderr, file=sys.stderr) return proc def runSudo(args): if isinstance(args, str): args = args.split(' ') child = subprocess.Popen(["sudo"] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if re.search('^\\[sudo\\] password for .+', child.stdout.read().strip()): child.communicate(input=input().encode())[0] child.wait() stderr = child.stderr.read().strip() if stderr != '': print(stderr, file=sys.stderr) return child def setCertPerms(service): dirs = ["/etc/letsencrypt", "/etc/letsencrypt/live", "/etc/letsencrypt/archive"] pkeyFile = env['certs'][service]['pkey'] domain_dir = re.search('.+(?=\\/.+$)', pkeyFile).group(0) dirs += [domain_dir, re.sub('live', 'archive', domain_dir)] for path in dirs: setOwner(path, 0, env['podman_uid']) setPerms(path, 711) setOwner(pkeyFile, 0, getUid(service)) setPerms(pkeyFile, 640) def setNftables(): print() renderFile("nftables.conf.mako") if not filecmp.cmp("nftables.conf.rendered", "/etc/nftables.conf"): print("nftables.conf changed, copying new version.") runSudo(["cp", "nftables.conf.rendered", "/etc/nftables.conf"]) runSudo(["systemctl", "restart", "nftables"]) else: print("nftables.conf unchanged.") def setOwner(path, uid=None, gid=None): stat = os.stat(path) if uid is None: uid = stat.st_uid if gid is None: gid = stat.st_gid if stat.st_uid != uid or stat.st_gid != gid: print(f"Changing ownership of {path} to {uid}:{gid} from {stat.st_uid}:{stat.st_gid}.") runSudo(["chown", f"{uid}:{gid}", path]) else: print(f"Ownership of {path} already set to {uid}:{gid}.") def setPerms(path, mode): mode = str(mode) stat = os.stat(path) curMode = oct(stat.st_mode)[-3:] if mode != curMode: print(f"Changing permissions of {path} to {mode} from {curMode}.") if stat.st_uid == os.getuid(): subprocess.run(["chmod", mode, path]) else: runSudo(["chmod", mode, path]) else: print(f"Permissions of {path} already set to {mode}.") def setupProj(project): print(f"Running setup for project {project}.") if os.path.isfile(f"projects/{project}/compose.yaml.rendered"): backupProj(project) if project == 'diun': if not os.path.isfile(env['socket']): if env['rootless']: subprocess.run(["systemctl", "--user", "restart", "podman.socket"]) else: runSudo("systemctl restart podman.socket") setPerms(env['socket'], 660) setOwner(env['socket'], env['podman_uid'], getUid('diun')) if project in env['certs'].keys(): setCertPerms(project) for templateFile in glob(f"projects/{project}/*.mako", include_hidden=True): renderFile(templateFile) renderedFilename = re.sub('\\.mako$', '.rendered', templateFile) setPerms(renderedFilename, 640) setOwner(renderedFilename, os.getuid(), getUid(project)) if project in env['volumes']: for volume in env['volumes'][project].values(): if not os.path.exists(volume): os.makedirs(volume) setPerms(volume, 750) setOwner(volume, getUid(project), env['podman_uid']) upProj(project) def upProj(project): if runPodman("pod", ["exists", f"pod_{project}"]).returncode == 0: print(f"Tearing down stack for project {project}.") runPodman("compose", ["-f", f"projects/{project}/compose.yaml.rendered", "down"]) print(f"Creating & starting stack for project {project}.") runPodman("compose", ["-f", f"projects/{project}/compose.yaml.rendered", "up", "-d"]) def updateProj(project): if not os.path.isfile(f"projects/{project}/compose.yaml.rendered"): setupProj(project) print(f"Running update for project {project}.") if len(pullProj(project)) > 0: backupProj(project) upProj(project) def main(): envFile = "pyenv.yml" secretsFile = "pysecrets.yml" os.chdir(os.path.realpath(sys.path[0])) with open(envFile, 'r') as envfile, open(secretsFile, 'r') as secretsfile: global env, secrets env = yaml.safe_load(Template(filename=envFile).render()) secrets = yaml.safe_load(secretsfile) setOwner(secretsFile, os.getuid(), os.getgid()) setPerms(secretsFile, 600) print("\nUsing socket " + env['socket'] + ".") action = '' if len(sys.argv) > 1: action = sys.argv[1] else: print("\nChoose action:") print("[1/S] Setup project") print("[2/U] Update project") print("[3/B] Backup project") while action == '': action = input("Action: ") projects = os.listdir("projects") print(f"\nProjects list: {projects}") if len(sys.argv) > 2: target_projects = sys.argv[2] else: target_projects = input("Target compose project(s), space separated, leave empty to target all: ") if target_projects.strip() == '': target_projects = projects else: target_projects = re.split(' ?, ?| ', target_projects.strip()) print(f"Target projects: {target_projects}") match action.casefold(): case '1' | 's' | 'setup': setNftables() for project in target_projects: try: print() setupProj(project) except Exception as e: print(e, file=sys.stderr) print(f"Failed to setup project {project}.", file=sys.stderr) case '2' | 'u' | 'update': for project in target_projects: try: print() updateProj(project) except Exception as e: print(e, file=sys.stderr) print(f"Failed to update project {project}.", file=sys.stderr) case '3' | 'b' | 'backup': print() if not os.path.exists(env['borg_repo']): print(f"Creating borg repository {env['borg_repo']}.") runBorg(["init", "--encryption", "repokey", env['borg_repo']]) borgCreate("secrets", path=secretsFile) for project in target_projects: try: print() backupProj(project) except Exception as e: print(e, file=sys.stderr) print(f"Failed to backup project {project}.", file=sys.stderr) runBorg(["compact", env['borg_repo']]) if __name__ == "__main__": main()