From d96e77704c06b86baa1db654c4d9e1e067ecb631 Mon Sep 17 00:00:00 2001 From: Kerstin Humm Date: Mon, 22 Jul 2024 17:11:49 +0200 Subject: [PATCH] nixos/weblate: init module and test Co-authored-by: Taeer Bar-Yam --- nixos/modules/module-list.nix | 1 + nixos/modules/services/web-apps/weblate.nix | 388 ++++++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/web-apps/weblate.nix | 104 ++++++ 4 files changed, 494 insertions(+) create mode 100644 nixos/modules/services/web-apps/weblate.nix create mode 100644 nixos/tests/web-apps/weblate.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 6b83c0bab4062..ad7f075788be8 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1484,6 +1484,7 @@ ./services/web-apps/trilium.nix ./services/web-apps/tt-rss.nix ./services/web-apps/vikunja.nix + ./services/web-apps/weblate.nix ./services/web-apps/whitebophir.nix ./services/web-apps/wiki-js.nix ./services/web-apps/windmill.nix diff --git a/nixos/modules/services/web-apps/weblate.nix b/nixos/modules/services/web-apps/weblate.nix new file mode 100644 index 0000000000000..398d634a2b928 --- /dev/null +++ b/nixos/modules/services/web-apps/weblate.nix @@ -0,0 +1,388 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.weblate; + + dataDir = "/var/lib/weblate"; + settingsDir = "${dataDir}/settings"; + + finalPackage = cfg.package.overridePythonAttrs (old: { + # We only support the PostgreSQL backend in this module + dependencies = old.dependencies ++ cfg.package.optional-dependencies.postgres; + # Use a settings module in dataDir, to avoid having to rebuild the package + # when user changes settings. + makeWrapperArgs = (old.makeWrapperArgs or [ ]) ++ [ + "--set PYTHONPATH \"${settingsDir}\"" + "--set DJANGO_SETTINGS_MODULE \"settings\"" + ]; + }); + inherit (finalPackage) python; + + pythonEnv = python.buildEnv.override { + extraLibs = with python.pkgs; [ + (toPythonModule finalPackage) + celery + ]; + }; + + # This extends and overrides the weblate/settings_example.py code found in upstream. + weblateConfig = + '' + # This was autogenerated by the NixOS module. + + SITE_TITLE = "Weblate" + SITE_DOMAIN = "${cfg.localDomain}" + # TLS terminates at the reverse proxy, but this setting controls how links to weblate are generated. + ENABLE_HTTPS = True + SESSION_COOKIE_SECURE = ENABLE_HTTPS + DATA_DIR = "${dataDir}" + CACHE_DIR = f"{DATA_DIR}/cache" + STATIC_ROOT = "${finalPackage.static}/static" + MEDIA_ROOT = "/var/lib/weblate/media" + COMPRESS_ROOT = "${finalPackage.static}/compressor-cache" + DEBUG = False + + DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": "/run/postgresql", + "NAME": "weblate", + "USER": "weblate", + } + } + + with open("${cfg.djangoSecretKeyFile}") as f: + SECRET_KEY = f.read().rstrip("\n") + + CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "unix://${config.services.redis.servers.weblate.unixSocket}", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "PASSWORD": None, + "CONNECTION_POOL_KWARGS": {}, + }, + "KEY_PREFIX": "weblate", + "TIMEOUT": 3600, + }, + "avatar": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/var/lib/weblate/avatar-cache", + "TIMEOUT": 86400, + "OPTIONS": {"MAX_ENTRIES": 1000}, + } + } + + + CELERY_TASK_ALWAYS_EAGER = False + CELERY_BROKER_URL = "redis+socket://${config.services.redis.servers.weblate.unixSocket}" + CELERY_RESULT_BACKEND = CELERY_BROKER_URL + + VCS_BACKENDS = ("weblate.vcs.git.GitRepository",) + + '' + + lib.optionalString cfg.smtp.enable '' + ADMINS = (("Weblate Admin", "${cfg.smtp.user}"),) + + EMAIL_HOST = "${cfg.smtp.host}" + EMAIL_USE_TLS = True + EMAIL_HOST_USER = "${cfg.smtp.user}" + SERVER_EMAIL = "${cfg.smtp.user}" + DEFAULT_FROM_EMAIL = "${cfg.smtp.user}" + EMAIL_PORT = 587 + with open("${cfg.smtp.passwordFile}") as f: + EMAIL_HOST_PASSWORD = f.read().rstrip("\n") + + '' + + cfg.extraConfig; + settings_py = + pkgs.runCommand "weblate_settings.py" + { + inherit weblateConfig; + passAsFile = [ "weblateConfig" ]; + } + '' + mkdir -p $out + cat \ + ${finalPackage}/${python.sitePackages}/weblate/settings_example.py \ + $weblateConfigPath \ + > $out/settings.py + ''; + + environment = { + PYTHONPATH = "${settingsDir}:${pythonEnv}/${python.sitePackages}/"; + DJANGO_SETTINGS_MODULE = "settings"; + # We run Weblate through gunicorn, so we can't utilise the env var set in the wrapper. + inherit (finalPackage) GI_TYPELIB_PATH; + }; + + weblatePath = with pkgs; [ + gitSVN + + #optional + git-review + tesseract + licensee + mercurial + ]; +in +{ + + options = { + services.weblate = { + enable = lib.mkEnableOption "Weblate service"; + + package = lib.mkPackageOption pkgs "weblate" { }; + + localDomain = lib.mkOption { + description = "The domain name serving your Weblate instance."; + example = "weblate.example.org"; + type = lib.types.str; + }; + + djangoSecretKeyFile = lib.mkOption { + description = '' + Location of the Django secret key. + + This should be a path pointing to a file with secure permissions (not /nix/store). + + Can be generated with `weblate-generate-secret-key` which is available as the `weblate` user. + ''; + type = lib.types.path; + }; + + extraConfig = lib.mkOption { + type = lib.types.lines; + default = ""; + description = '' + Text to append to `settings.py` Weblate configuration file. + ''; + }; + + smtp = { + enable = lib.mkEnableOption "Weblate SMTP support"; + user = lib.mkOption { + description = "SMTP login name."; + example = "weblate@example.org"; + type = lib.types.str; + }; + + host = lib.mkOption { + description = "SMTP host used when sending emails to users."; + type = lib.types.str; + example = "127.0.0.1"; + }; + + passwordFile = lib.mkOption { + description = '' + Location of a file containing the SMTP password. + + This should be a path pointing to a file with secure permissions (not /nix/store). + ''; + type = lib.types.path; + }; + }; + + }; + }; + + config = lib.mkIf cfg.enable { + + systemd.tmpfiles.rules = [ "L+ ${settingsDir} - - - - ${settings_py}" ]; + + services.nginx = { + enable = true; + virtualHosts."${cfg.localDomain}" = { + + forceSSL = true; + enableACME = true; + + locations = { + "= /favicon.ico".alias = "${finalPackage}/${python.sitePackages}/weblate/static/favicon.ico"; + "/static/".alias = "${finalPackage.static}/static/"; + "/static/CACHE/".alias = "${finalPackage.static}/compressor-cache/CACHE/"; + "/media/".alias = "/var/lib/weblate/media/"; + "/".proxyPass = "http://unix:///run/weblate.socket"; + }; + + }; + }; + + systemd.services.weblate-postgresql-setup = { + description = "Weblate PostgreSQL setup"; + after = [ "postgresql.service" ]; + serviceConfig = { + Type = "oneshot"; + User = "postgres"; + Group = "postgres"; + ExecStart = '' + ${config.services.postgresql.package}/bin/psql weblate -c "CREATE EXTENSION IF NOT EXISTS pg_trgm" + ''; + }; + }; + + systemd.services.weblate-migrate = { + description = "Weblate migration"; + after = [ "weblate-postgresql-setup.service" ]; + requires = [ "weblate-postgresql-setup.service" ]; + # We want this to be active on boot, not just on socket activation + wantedBy = [ "multi-user.target" ]; + inherit environment; + path = weblatePath; + serviceConfig = { + Type = "oneshot"; + StateDirectory = "weblate"; + User = "weblate"; + Group = "weblate"; + ExecStart = "${finalPackage}/bin/weblate migrate --noinput"; + }; + }; + + systemd.services.weblate-celery = { + description = "Weblate Celery"; + after = [ + "network.target" + "redis.service" + "postgresql.service" + ]; + # We want this to be active on boot, not just on socket activation + wantedBy = [ "multi-user.target" ]; + environment = environment // { + CELERY_WORKER_RUNNING = "1"; + }; + path = weblatePath; + # Recommendations from: + # https://github.com/WeblateOrg/weblate/blob/main/weblate/examples/celery-weblate.service + serviceConfig = + let + # We have to push %n through systemd's replacement, therefore %%n. + pidFile = "/run/celery/weblate-%%n.pid"; + nodes = "celery notify memory backup translate"; + cmd = verb: '' + ${pythonEnv}/bin/celery multi ${verb} \ + ${nodes} \ + -A "weblate.utils" \ + --pidfile=${pidFile} \ + --logfile=/var/log/celery/weblate-%%n%%I.log \ + --loglevel=DEBUG \ + --beat:celery \ + --queues:celery=celery \ + --prefetch-multiplier:celery=4 \ + --queues:notify=notify \ + --prefetch-multiplier:notify=10 \ + --queues:memory=memory \ + --prefetch-multiplier:memory=10 \ + --queues:translate=translate \ + --prefetch-multiplier:translate=4 \ + --concurrency:backup=1 \ + --queues:backup=backup \ + --prefetch-multiplier:backup=2 + ''; + in + { + Type = "forking"; + User = "weblate"; + Group = "weblate"; + WorkingDirectory = "${finalPackage}/${python.sitePackages}/weblate/"; + RuntimeDirectory = "celery"; + RuntimeDirectoryPreserve = "restart"; + LogsDirectory = "celery"; + ExecStart = cmd "start"; + ExecReload = cmd "restart"; + ExecStop = '' + ${pythonEnv}/bin/celery multi stopwait \ + ${nodes} \ + --pidfile=${pidFile} + ''; + Restart = "always"; + }; + }; + + systemd.services.weblate = { + description = "Weblate Gunicorn app"; + after = [ + "network.target" + "weblate-migrate.service" + "weblate-celery.service" + ]; + requires = [ + "weblate-migrate.service" + "weblate-celery.service" + "weblate.socket" + ]; + inherit environment; + path = weblatePath; + serviceConfig = { + Type = "notify"; + NotifyAccess = "all"; + ExecStart = + let + gunicorn = python.pkgs.gunicorn.overridePythonAttrs (old: { + # Allows Gunicorn to set a meaningful process name + dependencies = (old.dependencies or [ ]) ++ old.optional-dependencies.setproctitle; + }); + in + '' + ${gunicorn}/bin/gunicorn \ + --name=weblate \ + --bind='unix:///run/weblate.socket' \ + weblate.wsgi + ''; + ExecReload = "kill -s HUP $MAINPID"; + KillMode = "mixed"; + PrivateTmp = true; + WorkingDirectory = dataDir; + StateDirectory = "weblate"; + RuntimeDirectory = "weblate"; + User = "weblate"; + Group = "weblate"; + }; + }; + + systemd.sockets.weblate = { + before = [ "nginx.service" ]; + wantedBy = [ "sockets.target" ]; + socketConfig = { + ListenStream = "/run/weblate.socket"; + SocketUser = "weblate"; + SocketGroup = "weblate"; + SocketMode = "770"; + }; + }; + + services.redis.servers.weblate = { + enable = true; + user = "weblate"; + unixSocket = "/run/redis-weblate/redis.sock"; + unixSocketPerm = 770; + }; + + services.postgresql = { + enable = true; + ensureUsers = [ + { + name = "weblate"; + ensureDBOwnership = true; + } + ]; + ensureDatabases = [ "weblate" ]; + }; + + users.users.weblate = { + isSystemUser = true; + group = "weblate"; + packages = [ finalPackage ] ++ weblatePath; + }; + + users.groups.weblate.members = [ config.services.nginx.user ]; + }; + + meta.maintainers = with lib.maintainers; [ erictapen ]; + +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 12f8954c40403..8fadeb88f366e 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1072,6 +1072,7 @@ in { wastebin = handleTest ./wastebin.nix {}; watchdogd = handleTest ./watchdogd.nix {}; webhook = runTest ./webhook.nix; + weblate = handleTest ./web-apps/weblate.nix {}; wiki-js = handleTest ./wiki-js.nix {}; wine = handleTest ./wine.nix {}; wireguard = handleTest ./wireguard {}; diff --git a/nixos/tests/web-apps/weblate.nix b/nixos/tests/web-apps/weblate.nix new file mode 100644 index 0000000000000..40d60f7e5f996 --- /dev/null +++ b/nixos/tests/web-apps/weblate.nix @@ -0,0 +1,104 @@ +import ../make-test-python.nix ( + { pkgs, ... }: + + let + certs = import ../common/acme/server/snakeoil-certs.nix; + + serverDomain = certs.domain; + + admin = { + username = "admin"; + password = "snakeoilpass"; + }; + # An API token that we manually insert into the db as a valid one. + apiToken = "OVJh65sXaAfQMZ4NTcIGbFZIyBZbEZqWTi7azdDf"; + in + { + name = "weblate"; + meta.maintainers = with pkgs.lib.maintainers; [ erictapen ]; + + nodes.server = + { pkgs, lib, ... }: + { + virtualisation.memorySize = 2048; + + services.weblate = { + enable = true; + localDomain = "${serverDomain}"; + djangoSecretKeyFile = pkgs.writeText "weblate-django-secret" "thisissnakeoilsecretwithmorethan50characterscorrecthorsebatterystaple"; + extraConfig = '' + # Weblate tries to fetch Avatars from the network + ENABLE_AVATARS = False + ''; + }; + + services.nginx.virtualHosts."${serverDomain}" = { + enableACME = lib.mkForce false; + sslCertificate = certs."${serverDomain}".cert; + sslCertificateKey = certs."${serverDomain}".key; + }; + + security.pki.certificateFiles = [ certs.ca.cert ]; + + networking.hosts."::1" = [ "${serverDomain}" ]; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; + + users.users.weblate.shell = pkgs.bashInteractive; + }; + + nodes.client = + { pkgs, nodes, ... }: + { + environment.systemPackages = [ pkgs.wlc ]; + + environment.etc."xdg/weblate".text = '' + [weblate] + url = https://${serverDomain}/api/ + key = ${apiToken} + ''; + + networking.hosts."${nodes.server.networking.primaryIPAddress}" = [ "${serverDomain}" ]; + + security.pki.certificateFiles = [ certs.ca.cert ]; + }; + + testScript = '' + import json + + start_all() + server.wait_for_unit("weblate.socket") + server.wait_until_succeeds("curl -f https://${serverDomain}/") + server.succeed("sudo -iu weblate -- weblate createadmin --username ${admin.username} --password ${admin.password} --email weblate@example.org") + + # It's easier to replace the generated API token with a predefined one than + # to extract it at runtime. + server.succeed("sudo -iu weblate -- psql -d weblate -c \"UPDATE authtoken_token SET key = '${apiToken}' WHERE user_id = (SELECT id FROM weblate_auth_user WHERE username = 'admin');\"") + + client.wait_for_unit("multi-user.target") + + # Test the official Weblate client wlc. + client.wait_until_succeeds("REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt wlc --debug list-projects") + + def call_wl_api(arg): + (rv, result) = client.execute("curl -H \"Content-Type: application/json\" -H \"Authorization: Token ${apiToken}\" https://${serverDomain}/api/{}".format(arg)) + assert rv == 0 + print(result) + + call_wl_api("users/ --data '{}'".format( + json.dumps( + {"username": "test1", + "full_name": "test1", + "email": "test1@example.org" + }))) + + # TODO: Check sending and receiving email. + # server.wait_for_unit("postfix.service") + + # TODO: The goal is for this to succeed, but there are still some checks failing. + # server.succeed("sudo -iu weblate -- weblate check --deploy") + ''; + } +)