vps/manage.py

447 lines
13 KiB
Python
Raw Normal View History

#!/usr/bin/python3
import os, sys, re, shutil, time
import filecmp
from glob import glob
from mako.template import Template
2024-10-07 16:57:45 +02:00
from pathlib import Path
import subprocess
import yaml
def backupProj(project):
database = None
paths = []
2024-10-15 09:41:51 +02:00
sqlite_db = None
2024-10-14 17:38:10 +02:00
if project == 'postgres':
database = 'all'
elif project in secrets['postgres'].keys():
database = project
if project in env['backup'].keys():
for path in env['backup'][project]:
paths += [path]
2024-10-15 09:41:51 +02:00
if project in env['backup_sqlite'].keys():
sqlite_db = env['backup_sqlite'][project]
if database is not None or sqlite_db is not None or len(paths) > 0:
print(f"Running backup for project {project}.")
if len(paths) == 0:
2024-10-15 09:41:51 +02:00
borgCreate(project, database=database, sqlite_db=sqlite_db)
else:
2024-10-15 09:41:51 +02:00
borgCreate(project, path=paths[0], database=database, sqlite_db=sqlite_db)
for path in paths[1:]:
archiveName = re.sub('/', '-', path)
borgCreate(f"{project}-{archiveName}", path=path)
2024-10-15 09:41:51 +02:00
def borgCreate(name, path=None, database=None, sqlite_db=None):
if path is None and database is None and sqlite_db is None:
print(f"Cannot create backup, you must pass at least one parameter amongst: path, database, database_path (archive name = {name}).", file=sys.stderr)
return 1
borgPaths = []
2024-10-07 16:57:45 +02:00
if path is not None:
absPath = Path(path).absolute()
parentDir = absPath.parent.absolute()
os.chdir(parentDir)
borgPaths += [absPath.relative_to(parentDir)]
borgInput = None
if database is not None:
print(f"Dumping database {database}.")
2024-10-07 16:57:45 +02:00
2024-10-14 17:38:10 +02:00
if database == 'all':
dumpProc = subprocess.run(["podman", "exec", "postgres", "pg_dumpall"], capture_output=True, text=True)
else:
dumpProc = subprocess.run(["podman", "exec", "postgres", "pg_dump", "-c", database], capture_output=True, text=True)
if dumpProc.returncode != 0:
print(f"Failed to dump database {database}.", file=sys.stderr)
return 1
borgPaths += ["-", "--stdin-name", f"{database}.sql"]
borgInput = dumpProc.stdout
2024-10-15 09:41:51 +02:00
if sqlite_db is not None:
print(f"Dumping SQLite database {sqlite_db}.")
sqlite_args = ["podman", "run", "--rm", "-v", f"{sqlite_db}:/db.sqlite", "docker.io/library/alpine:latest", "sh", "-c", "apk add sqlite &>/dev/null; sqlite3 /db.sqlite .dump"]
dumpProc = subprocess.run(sqlite_args, capture_output=True, text=True)
if dumpProc.returncode != 0:
print(f"Failed to dump SQLite database {sqlite_db}.", file=sys.stderr)
return 1
database = Path(sqlite_db).stem
borgPaths += ["-", "--stdin-name", f"{database}.sqlite"]
borgInput = dumpProc.stdout
if path is None:
print(f"Creating archive '{name}' from database {database}.")
elif database is None:
print(f"Creating archive '{name}' from path {path}.")
else:
print(f"Creating archive '{name}' from database {database} and path {absPath}.")
borgArgs = ["create", "--compression=lzma", f"{env['borg_repo']}::{name}_{{utcnow}}"]
runBorg(borgArgs + borgPaths, input=borgInput)
2024-10-07 16:57:45 +02:00
os.chdir(os.path.realpath(sys.path[0]))
print(f"Pruning archives '{name}_*'.")
runBorg(["prune", "--glob-archives", f"{name}_*"] + env['borg_prune_opts'] + [env['borg_repo']])
2024-10-07 16:57:45 +02:00
def getImageId (image):
2024-10-07 11:37:52 +02:00
return runPodman("image", ["inspect", "--format", "{{.Id}}", image]).stdout.strip()
def getUid(service):
2024-10-15 10:23:53 +02:00
if service in env['users'].keys() and env['users'][service] != 0:
2024-10-07 11:37:52 +02:00
return env['users'][service] + env['uid_shift']
else:
2024-10-07 11:37:52 +02:00
return env['podman_uid']
2024-10-15 21:44:02 +02:00
def promptTargetProjects():
projects = os.listdir("projects")
print(f"\nProjects list: {projects}")
if len(sys.argv) > 2:
target_projects = sys.argv[2]
else:
target_projects = input("Target compose project(s), space separated, leave empty to target all: ")
if target_projects.strip() == '':
target_projects = projects
else:
target_projects = re.split(' ?, ?| ', target_projects.strip())
print(f"Target projects: {target_projects}")
return target_projects
def pullProj(project):
print(f"Pulling images for project {project}.")
with open(f"projects/{project}/compose.yaml.rendered", 'r') as composefile:
images = re.findall('(?<=image:\\s).+', composefile.read())
pulledImages = []
for image in images:
currentId = getImageId(image)
2024-10-15 11:22:12 +02:00
if re.search('^localhost/', image):
runPodman("compose", ["-f", f"projects/{project}/compose.yaml.rendered", "build", "--pull"])
else:
runPodman("pull", image)
pulledId = getImageId(image)
if currentId != pulledId:
pulledImages += image
print(f"Pulled new version of image {image}.")
else:
print(f"No update available for image {image}.")
return pulledImages
2024-10-15 21:44:02 +02:00
def renderFile(templateFile, renderedFile=None):
if renderedFile is None:
renderedFile = re.sub('\\.mako$', '.rendered', templateFile)
2024-10-15 21:44:02 +02:00
print(f"Rendering file {templateFile} to {renderedFile}.")
template = Template(filename=templateFile)
2024-10-15 21:44:02 +02:00
with open(renderedFile, "w") as outputFile:
outputFile.write(template.render(env=env, secrets=secrets))
def runBorg(args, input=None):
2024-10-07 16:57:45 +02:00
if isinstance(args, str):
args = args.split(' ')
borgEnv = {"BORG_PASSPHRASE": secrets['borg']}
proc = subprocess.run(["borg"] + args, input=input, capture_output=True, text=True, env=borgEnv)
2024-10-07 16:57:45 +02:00
stderr = proc.stderr.strip()
2024-10-07 16:57:45 +02:00
if stderr != '':
print(stderr, file=sys.stderr)
return proc
2024-10-07 16:57:45 +02:00
2024-10-07 11:37:52 +02:00
def runPodman(cmd, args):
if isinstance(args, str):
args = args.split(' ')
if cmd == 'compose':
runArgs = ["podman-compose"] + args
else:
runArgs = ["podman", cmd] + args
if env['rootless']:
proc = subprocess.run(runArgs, capture_output=True, text=True)
stderr = proc.stderr.strip()
2024-10-07 11:37:52 +02:00
else:
proc = runSudo(runArgs)
stderr = proc.stderr.read().strip()
if stderr != '':
print(stderr, file=sys.stderr)
return proc
2024-10-07 11:37:52 +02:00
def runSudo(args):
if isinstance(args, str):
args = args.split(' ')
child = subprocess.Popen(["sudo"] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
2024-10-15 21:44:02 +02:00
stdout = child.stdout.read().strip()
if stdout != '':
print(stdout)
if re.search('^\\[sudo\\] password for .+', stdout):
2024-10-07 11:37:52 +02:00
child.communicate(input=input().encode())[0]
child.wait()
2024-10-15 21:44:02 +02:00
stdout = child.stdout.read().strip()
if stdout != '':
print(stdout)
2024-10-07 11:37:52 +02:00
stderr = child.stderr.read().strip()
if stderr != '':
print(stderr, file=sys.stderr)
return child
def setCertPerms(service):
2024-10-07 11:37:52 +02:00
dirs = ["/etc/letsencrypt", "/etc/letsencrypt/live", "/etc/letsencrypt/archive"]
pkeyFile = env['certs'][service]['pkey']
2024-10-07 11:37:52 +02:00
domain_dir = re.search('.+(?=\\/.+$)', pkeyFile).group(0)
2024-10-07 11:37:52 +02:00
dirs += [domain_dir, re.sub('live', 'archive', domain_dir)]
for path in dirs:
setOwner(path, 0, env['podman_uid'])
setPerms(path, 711)
setOwner(pkeyFile, 0, getUid(service))
setPerms(pkeyFile, 640)
def setNftables():
2024-10-07 16:57:45 +02:00
print()
renderFile("nftables.conf.mako")
if not filecmp.cmp("nftables.conf.rendered", "/etc/nftables.conf"):
print("nftables.conf changed, copying new version.")
2024-10-07 11:37:52 +02:00
runSudo(["cp", "nftables.conf.rendered", "/etc/nftables.conf"])
runSudo(["systemctl", "restart", "nftables"])
else:
print("nftables.conf unchanged.")
def setOwner(path, uid=None, gid=None):
stat = os.stat(path)
2024-10-01 13:57:43 +02:00
if uid is None:
uid = stat.st_uid
2024-10-01 13:57:43 +02:00
if gid is None:
gid = stat.st_gid
if stat.st_uid != uid or stat.st_gid != gid:
print(f"Changing ownership of {path} to {uid}:{gid} from {stat.st_uid}:{stat.st_gid}.")
2024-10-07 11:37:52 +02:00
runSudo(["chown", f"{uid}:{gid}", path])
else:
print(f"Ownership of {path} already set to {uid}:{gid}.")
def setPerms(path, mode):
mode = str(mode)
stat = os.stat(path)
curMode = oct(stat.st_mode)[-3:]
if mode != curMode:
print(f"Changing permissions of {path} to {mode} from {curMode}.")
2024-10-07 11:37:52 +02:00
if stat.st_uid == os.getuid():
subprocess.run(["chmod", mode, path])
else:
2024-10-07 11:37:52 +02:00
runSudo(["chmod", mode, path])
else:
print(f"Permissions of {path} already set to {mode}.")
2024-10-15 21:44:02 +02:00
def setupNGINX():
if os.getuid() != 0:
print("Current user is not root, rerunning program as root.")
runSudo([sys.executable, sys.argv[0], "nginx"])
return
for dir in ["sites-enabled", "snippets"]:
print(f"Recreating directory /etc/nginx/{dir}")
shutil.rmtree(f"/etc/nginx/{dir}")
os.mkdir(f"/etc/nginx/{dir}")
for templateFile in glob("nginx/**/*.conf", recursive=True, include_hidden=True):
renderFile(templateFile, "/etc/" + templateFile)
subprocess.run(["nginx", "-t"])
print("Reloading service nginx.")
runSudo("systemctl reload nginx")
def setupProj(project):
print(f"Running setup for project {project}.")
if os.path.isfile(f"projects/{project}/compose.yaml.rendered"):
backupProj(project)
2024-10-07 11:37:52 +02:00
if project == 'diun':
if not os.path.isfile(env['socket']):
if env['rootless']:
subprocess.run(["systemctl", "--user", "restart", "podman.socket"])
else:
runSudo("systemctl restart podman.socket")
setPerms(env['socket'], 660)
setOwner(env['socket'], env['podman_uid'], getUid('diun'))
2024-10-07 11:37:52 +02:00
if project in env['certs'].keys():
setCertPerms(project)
for templateFile in glob(f"projects/{project}/*.mako", include_hidden=True):
renderFile(templateFile)
renderedFilename = re.sub('\\.mako$', '.rendered', templateFile)
setPerms(renderedFilename, 640)
2024-10-07 11:37:52 +02:00
setOwner(renderedFilename, os.getuid(), getUid(project))
2024-10-07 14:42:58 +02:00
if project in env['volumes']:
for volume in env['volumes'][project].values():
if not os.path.exists(volume):
os.makedirs(volume)
2024-10-07 14:42:58 +02:00
setPerms(volume, 750)
setOwner(volume, getUid(project), env['podman_uid'])
2024-10-07 14:42:58 +02:00
upProj(project)
def upProj(project):
if runPodman("pod", ["exists", f"pod_{project}"]).returncode == 0:
2024-10-07 11:37:52 +02:00
print(f"Tearing down stack for project {project}.")
runPodman("compose", ["-f", f"projects/{project}/compose.yaml.rendered", "down"])
time.sleep(0.25) # Give time to pasta to unbind port
2024-10-07 11:37:52 +02:00
print(f"Creating & starting stack for project {project}.")
2024-10-07 11:37:52 +02:00
runPodman("compose", ["-f", f"projects/{project}/compose.yaml.rendered", "up", "-d"])
def updateProj(project):
if not os.path.isfile(f"projects/{project}/compose.yaml.rendered"):
setupProj(project)
print(f"Running update for project {project}.")
if len(pullProj(project)) > 0:
backupProj(project)
upProj(project)
def main():
2024-10-16 18:48:51 +02:00
envFile = "env.yml"
secretsFile = "secrets.yml"
os.chdir(os.path.realpath(sys.path[0]))
with open(envFile, 'r') as envfile, open(secretsFile, 'r') as secretsfile:
global env, secrets
2024-10-07 11:37:52 +02:00
env = yaml.safe_load(Template(filename=envFile).render())
secrets = yaml.safe_load(secretsfile)
2024-10-07 11:37:52 +02:00
2024-10-15 21:44:02 +02:00
if os.getuid() != 0:
setOwner(secretsFile, os.getuid(), os.getgid())
setPerms(secretsFile, 600)
2024-10-07 11:37:52 +02:00
print("\nUsing socket " + env['socket'] + ".")
action = ''
if len(sys.argv) > 1:
action = sys.argv[1]
else:
print("\nChoose action:")
print("[1/S] Setup project")
print("[2/U] Update project")
print("[3/B] Backup project")
2024-10-15 21:44:02 +02:00
print("[4/N] Setup NGINX")
while action == '':
action = input("Action: ")
match action.casefold():
case '1' | 's' | 'setup':
2024-10-15 21:44:02 +02:00
target_projects = promptTargetProjects()
2024-10-07 16:57:45 +02:00
setNftables()
for project in target_projects:
try:
print()
setupProj(project)
except Exception as e:
print(e, file=sys.stderr)
print(f"Failed to setup project {project}.", file=sys.stderr)
case '2' | 'u' | 'update':
2024-10-15 21:44:02 +02:00
target_projects = promptTargetProjects()
for project in target_projects:
try:
print()
updateProj(project)
except Exception as e:
print(e, file=sys.stderr)
print(f"Failed to update project {project}.", file=sys.stderr)
case '3' | 'b' | 'backup':
2024-10-15 21:44:02 +02:00
target_projects = promptTargetProjects()
2024-10-07 16:57:45 +02:00
print()
if not os.path.exists(env['borg_repo']):
print(f"Creating borg repository {env['borg_repo']}.")
runBorg(["init", "--encryption", "repokey", env['borg_repo']])
borgCreate("secrets", path=secretsFile)
for project in target_projects:
try:
print()
backupProj(project)
except Exception as e:
print(e, file=sys.stderr)
print(f"Failed to backup project {project}.", file=sys.stderr)
runBorg(["compact", env['borg_repo']])
2024-10-15 21:44:02 +02:00
case '4' | 'n' | 'nginx':
setupNGINX()
if __name__ == "__main__":
main()