Changed how all docker images have their version set to use the compose repo

ansible/inventories/proxmox.yml

@@ -0,0 +1,10 @@
+proxmox:
+  hosts:
+    pve-goose:
+      ansible_host: 192.168.7.240
+    pve-duck:
+      ansible_host: 192.168.7.241
+
+  vars:
+    ansible_user: "ansible"
+    ansible_ssh_private_key_file: "/home/nick/.ssh/infra_ansible_key"

ansible/plays/machine_setup/domain-watchdog.yml

@@ -24,21 +24,28 @@
 
     - name: Deploy Domain Watchdog
       vars:
+        # Keep this path: compose prefixes named volumes (database_data etc.)
+        # with the directory name, so moving it orphans the existing data.
         domain_watchdog_dir: /opt/domain-watchdog
+        # Branch of AtlasHorizon/compose to deploy from; override with
+        # `-e compose_branch=<branch>` to test unmerged changes.
+        compose_branch: main
       block:
-        - name: Clone domain-watchdog repo
-          ansible.builtin.git:
-            repo: https://github.com/maelgangloff/domain-watchdog.git
-            dest: "{{ domain_watchdog_dir }}"
-            version: master
-            update: yes
-
         - name: Create public/content directory
           ansible.builtin.file:
             path: "{{ domain_watchdog_dir }}/public/content"
             state: directory
             mode: "0755"
 
+        # force: true so changes in the compose repo are picked up on reruns;
+        # it only reports changed when the content differs.
+        - name: Download docker-compose.yml
+          ansible.builtin.get_url:
+            url: "https://git.atlashorizon.net/AtlasHorizon/compose/raw/branch/{{ compose_branch }}/domain-watchdog/docker-compose.yml"
+            dest: "{{ domain_watchdog_dir }}/docker-compose.yml"
+            mode: "0644"
+            force: true
+
         # Pulled from Infisical, not generated/persisted to disk. Run this
         # playbook as `infisical run -- ansible-playbook -i inventory ...`
         # with DOMAIN_WATCHDOG_POSTGRES_PASSWORD, DOMAIN_WATCHDOG_APP_SECRET,

ansible/plays/machine_setup/grist.yml

@@ -27,6 +27,9 @@
         grist_dir: /opt/grist
         grist_hostname: grist.atlashorizon.net
         grist_oidc_issuer_host: auth.atlashorizon.net
+        # Branch of AtlasHorizon/compose to deploy from; override with
+        # `-e compose_branch=<branch>` to test unmerged changes.
+        compose_branch: main
       block:
         - name: Create persist directory
           ansible.builtin.file:
@@ -34,11 +37,14 @@
             state: directory
             mode: "0755"
 
-        - name: Deploy docker-compose.yml
+        # force: true so changes in the compose repo are picked up on reruns;
-          ansible.builtin.copy:
+        # it only reports changed when the content differs.
-            src: templates/grist-docker-compose.yml
+        - name: Download docker-compose.yml
+          ansible.builtin.get_url:
+            url: "https://git.atlashorizon.net/AtlasHorizon/compose/raw/branch/{{ compose_branch }}/grist/docker-compose.yml"
             dest: "{{ grist_dir }}/docker-compose.yml"
             mode: "0644"
+            force: true
 
         # Pulled from Infisical, not generated/persisted to disk. Run this
         # playbook as `infisical run -- ansible-playbook -i inventory ...`

ansible/plays/machine_setup/nametag.yml

@@ -24,21 +24,28 @@
 
     - name: Deploy Nametag
       vars:
+        # Keep this path: compose prefixes named volumes (postgres_data etc.)
+        # with the directory name, so moving it orphans the existing data.
         nametag_dir: /opt/nametag
+        # Branch of AtlasHorizon/compose to deploy from; override with
+        # `-e compose_branch=<branch>` to test unmerged changes.
+        compose_branch: main
       block:
-        - name: Clone nametag repo
-          ansible.builtin.git:
-            repo: https://github.com/mattogodoy/nametag.git
-            dest: "{{ nametag_dir }}"
-            version: master
-            update: yes
-
         - name: Create backups directory
           ansible.builtin.file:
             path: "{{ nametag_dir }}/backups"
             state: directory
             mode: "0755"
 
+        # force: true so changes in the compose repo are picked up on reruns;
+        # it only reports changed when the content differs.
+        - name: Download docker-compose.yml
+          ansible.builtin.get_url:
+            url: "https://git.atlashorizon.net/AtlasHorizon/compose/raw/branch/{{ compose_branch }}/nametag/docker-compose.yml"
+            dest: "{{ nametag_dir }}/docker-compose.yml"
+            mode: "0644"
+            force: true
+
         # Pulled from Infisical, not generated/persisted to disk. Run this
         # playbook as `infisical run -- ansible-playbook -i inventory ...`
         # with NAMETAG_DB_PASSWORD, NAMETAG_NEXTAUTH_SECRET,

ansible/plays/machine_setup/templates/omnivore.env.j2

@@ -1,72 +0,0 @@
-# Managed by Ansible (omnivore.yml) - generated secrets persist in
-# ansible/plays/machine_setup/credentials/omnivore/ on the control node.
-
-# Postgres & Migrate
-
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD={{ omnivore_postgres_password }}
-POSTGRES_DB=omnivore
-PGPASSWORD={{ omnivore_postgres_password }}
-PG_REPLICATION=false
-
-PG_HOST=postgres
-PG_PASSWORD={{ omnivore_pg_password }}
-PG_DB=omnivore
-PG_USER=app_user
-PG_PORT=5432
-PG_POOL_MAX=20
-
-# API
-
-API_ENV=local
-IMAGE_PROXY_SECRET={{ omnivore_image_proxy_secret }}
-JWT_SECRET={{ omnivore_jwt_secret }}
-SSO_JWT_SECRET={{ omnivore_sso_jwt_secret }}
-GATEWAY_URL=http://api:8080/api
-CONTENT_FETCH_URL=http://content-fetch:8080/?token={{ omnivore_verification_token }}
-GCS_USE_LOCAL_HOST=true
-GCS_UPLOAD_BUCKET=omnivore
-AUTO_VERIFY=true
-# Minio credentials are hardcoded in docker-compose.yml for the minio and
-# createbuckets services, so these must match those literal values.
-AWS_ACCESS_KEY_ID=minio
-AWS_SECRET_ACCESS_KEY=miniominio
-AWS_REGION=us-east-1
-CONTENT_FETCH_QUEUE_ENABLED=true
-
-IMAGE_PROXY_URL=http://{{ ansible_host }}:7070
-CLIENT_URL=http://{{ ansible_host }}:3000
-LOCAL_MINIO_URL=http://{{ ansible_host }}:1010
-
-# Redis
-REDIS_URL=redis://redis:6379/0
-
-# Mail
-WATCHER_API_KEY={{ omnivore_mail_watcher_key }}
-LOCAL_EMAIL_DOMAIN=domain.tld
-# SNS_ARN=arn_of_sns #for if you use SES and SNS for Email.
-
-# Web
-APP_ENV=prod
-BASE_URL=http://{{ ansible_host }}:3000
-SERVER_BASE_URL=http://{{ ansible_host }}:4000
-HIGHLIGHTS_BASE_URL=http://{{ ansible_host }}:3000
-# USE_DISCOVER_AI=true
-# OPENAI_API_KEY=<OPENAI_API_KEY>
-
-# Content Fetch
-VERIFICATION_TOKEN={{ omnivore_verification_token }}
-REST_BACKEND_ENDPOINT=http://api:8080/api
-SKIP_UPLOAD_ORIGINAL=true
-
-# Minio
-MINIO_ACCESS_KEY=minio
-MINIO_SECRET_KEY=miniominio
-AWS_S3_ENDPOINT_URL=http://minio:9000
-
-# Export
-EXPORT_QUEUE_NAME=omnivore-backend-queue
-
-# Youtube
-YOUTUBE_TRANSCRIPT_PROMPT=Reformat the supplied transcript data adding formatting, puncuation, and paragraphs, but do not summarize or change the content. Format the output as markdown.
-YOUTUBE_MAXIMUM_VIDEO_DURATION_TRANSCRIPT=1801

backend.tf

@@ -8,9 +8,9 @@ terraform {
   backend "s3" {
     bucket = "terraform"
     key    = "infra/terraform.tfstate"
-    region = "garage"
+    region = "us-east-1"
 
-    endpoint         = "http://s3.lan.oswaldtheotter.net:30188"
+    endpoint         = "https://goldie.lan.atlashorizon.net:30292"
     use_path_style = true
 
     skip_credentials_validation = true

butane/ai.yaml.tftpl

@@ -8,9 +8,12 @@ systemd:
       contents: |
         [Unit]
         Description=Gitea MCP Server
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force gitea-mcp
         # The image ships a broken HEALTHCHECK: it is CMD-SHELL form but the
@@ -21,7 +24,7 @@ systemd:
         ExecStart=/usr/bin/docker run --name gitea-mcp --no-healthcheck \
           -e GITEA_HOST=https://git.atlashorizon.net \
           -p 8080:8080 \
-          docker.gitea.com/gitea-mcp-server \
+          $${GITEA_MCP_IMAGE} \
           /app/gitea-mcp -t http --port 8080
         ExecStop=/usr/bin/docker stop gitea-mcp
         Restart=always
@@ -33,9 +36,12 @@ systemd:
       contents: |
         [Unit]
         Description=LiteLLM proxy
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force litellm
         ExecStart=/usr/bin/docker run --name litellm \
@@ -47,7 +53,7 @@ systemd:
           -e JSON_LOGS=True \
           -v /etc/litellm/config.yaml:/app/config.yaml:ro \
           -p 4000:4000 \
-          docker.litellm.ai/berriai/litellm:main-stable \
+          $${LITELLM_IMAGE} \
           --config /app/config.yaml --port 4000
         ExecStop=/usr/bin/docker stop litellm
         Restart=always
@@ -77,15 +83,18 @@ systemd:
       contents: |
         [Unit]
         Description=Qdrant vector database
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         # Both containers sit on a private docker network and Qdrant publishes
         # no ports: it ships with no authentication, and nothing outside this
         # host needs to reach it. Creation is idempotent, hence the "-".
         ExecStartPre=-/usr/bin/docker network create openwebui
-        ExecStartPre=/usr/bin/docker pull qdrant/qdrant:v1.19.1
+        ExecStartPre=/usr/bin/docker pull $${QDRANT_IMAGE}
         ExecStartPre=-/usr/bin/docker rm --force qdrant
         # Vectors live on the VM disk, so a taint destroys them. That is
         # deliberate -- they are derived data, rebuildable from the source
@@ -95,7 +104,7 @@ systemd:
         ExecStart=/usr/bin/docker run --name qdrant \
           --network openwebui \
           -v /var/lib/qdrant:/qdrant/storage \
-          qdrant/qdrant:v1.19.1
+          $${QDRANT_IMAGE}
         ExecStop=/usr/bin/docker stop qdrant
         Restart=always
         RestartSec=5s
@@ -119,12 +128,15 @@ systemd:
       contents: |
         [Unit]
         Description=Open Terminal sandbox
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker network create openwebui
-        ExecStartPre=/usr/bin/docker pull ghcr.io/open-webui/open-terminal:0.14.0
+        ExecStartPre=/usr/bin/docker pull $${OPEN_TERMINAL_IMAGE}
         ExecStartPre=-/usr/bin/docker rm --force open-terminal
         # A named volume, not a bind mount: /home/user belongs to the image's
         # unprivileged user, and a named volume inherits that ownership from
@@ -135,7 +147,7 @@ systemd:
           --network openwebui \
           -v open-terminal:/home/user \
           -e OPEN_TERMINAL_API_KEY=${openwebui_terminal_api_key} \
-          ghcr.io/open-webui/open-terminal:0.14.0
+          $${OPEN_TERMINAL_IMAGE}
         ExecStop=/usr/bin/docker stop open-terminal
         Restart=always
         RestartSec=5s
@@ -148,13 +160,16 @@ systemd:
         Description=Open WebUI
         # After= orders the start only, it does not wait for Qdrant to be
         # serving. Restart=always covers the gap if Qdrant is still coming up.
-        After=docker.service litellm.service qdrant.service open-terminal.service
+        After=docker.service litellm.service qdrant.service open-terminal.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         # Large image, so it is pulled up front: a slow first pull is then not
         # mistaken for a hung unit.
-        ExecStartPre=/usr/bin/docker pull ghcr.io/open-webui/open-webui:v0.11.4
+        ExecStartPre=/usr/bin/docker pull $${OPEN_WEBUI_IMAGE}
         ExecStartPre=-/usr/bin/docker network create openwebui
         ExecStartPre=-/usr/bin/docker rm --force open-webui
         # No volume is mounted. Every piece of durable state lives off-box --
@@ -193,50 +208,64 @@ systemd:
           -e OAUTH_PICTURE_CLAIM=avatar \
           -e "OAUTH_SCOPES=openid email profile groups avatar" \
           -e OAUTH_MERGE_ACCOUNTS_BY_EMAIL=true \
+          -e CORS_ALLOW_ORIGIN=https://llm.atlashorizon.net \
           -e WEBUI_URL=https://llm.atlashorizon.net \
           -e 'TERMINAL_SERVER_CONNECTIONS=[{"id":"local","name":"Open Terminal","url":"http://open-terminal:8000","key":"${openwebui_terminal_api_key}","auth_type":"bearer","enabled":true}]' \
           -p 3000:8080 \
-          ghcr.io/open-webui/open-webui:v0.11.4
+          $${OPEN_WEBUI_IMAGE}
         ExecStop=/usr/bin/docker stop open-webui
         Restart=always
         RestartSec=5s
         [Install]
         WantedBy=multi-user.target
-    # OpenCode headless server for n8n (and anything else on the LAN).
+    # OpenCode, run headless as a coding agent that n8n drives over HTTP.
     #
-    # Prerequisites:
+    # Prerequisites, all of which must exist before the VM is rebuilt:
-    #   - OPENCODE_SERVER_PASSWORD in Infisical (see secrets.tmpl)
+    #   - OPENCODE_SERVER_PASSWORD in Infisical (see secrets.tmpl). Every
-    #   - model keys in /etc/opencode/opencode.json matching LiteLLM
+    #     Butane file is rendered on every plan, so until the secret exists
-    #     model_name values (Admin UI or GET /v1/models). Models live in
+    #     no plan succeeds -- not just this VM's
-    #     the LiteLLM database, so this file cannot discover them.
+    #   - each model listed in /etc/opencode/opencode.json (below) defined
+    #     in LiteLLM under the same model_name
     #
-    # Listens on :4096 with HTTP Basic Auth (user "opencode"). n8n on
+    # Unlike Open Terminal, this publishes a port: n8n runs on oswald-runner,
-    # oswald-runner reaches it at http://192.168.7.61:4096 -- install the
+    # so the private "openwebui" network is out of its reach. That leaves HTTP
-    # n8n-nodes-opencode-ai community package, or use HTTP Request nodes
+    # Basic Auth (user "opencode") as the only thing between the LAN and an
-    # against the OpenAPI at /doc. LLM calls go through LiteLLM on :4000
+    # agent with a shell, and over plain HTTP. The agent runs as root inside
-    # using the same host-IP pattern as Open WebUI.
+    # the container, with the same outbound network access as Open Terminal.
+    #
+    # From n8n: http://192.168.7.61:4096, with the n8n-nodes-opencode-ai
+    # community node or HTTP Request nodes (the OpenAPI spec is at /doc).
     - name: opencode.service
       enabled: true
       contents: |
         [Unit]
         Description=OpenCode server
-        After=docker.service litellm.service
+        After=docker.service litellm.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
-        ExecStartPre=/usr/bin/docker pull ghcr.io/anomalyco/opencode:1.18.32
+        ExecStartPre=/usr/bin/docker pull $${OPENCODE_IMAGE}
         ExecStartPre=-/usr/bin/docker rm --force opencode
-        # /var/lib/opencode is the project workspace the agent edits. It
+        # Both directories are on the VM disk, so they survive container
-        # survives container restarts; a VM taint wipes it (scratch space).
+        # restarts but not a taint. workspace is what the agent edits; data
+        # holds its sessions, which n8n refers to by id across executions.
+        #
+        # The LiteLLM key is passed in the environment and read by the config
+        # file as {env:LITELLM_API_KEY}, which keeps the file free of secrets.
         ExecStart=/usr/bin/docker run --name opencode \
           -e OPENCODE_SERVER_USERNAME=opencode \
           -e OPENCODE_SERVER_PASSWORD=${opencode_server_password} \
+          -e LITELLM_API_KEY=${litellm_master_key} \
           -e OPENCODE_CONFIG=/etc/opencode/opencode.json \
           -v /etc/opencode/opencode.json:/etc/opencode/opencode.json:ro \
-          -v /var/lib/opencode:/workspace \
+          -v /var/lib/opencode/workspace:/workspace \
+          -v /var/lib/opencode/data:/root/.local/share/opencode \
           -w /workspace \
           -p 4096:4096 \
-          ghcr.io/anomalyco/opencode:1.18.32 \
+          $${OPENCODE_IMAGE} \
           serve --hostname 0.0.0.0 --port 4096
         ExecStop=/usr/bin/docker stop opencode
         Restart=always
@@ -267,15 +296,18 @@ systemd:
       contents: |
         [Unit]
         Description=Dozzle Agent
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force dozzle-agent
         ExecStart=/usr/bin/docker run --name dozzle-agent \
           -v /var/run/docker.sock:/var/run/docker.sock \
           -p 7007:7007 \
-          amir20/dozzle:v11.1.1 agent
+          $${DOZZLE_IMAGE} agent
         ExecStop=/usr/bin/docker stop dozzle-agent
         Restart=always
         RestartSec=5s
@@ -289,29 +321,13 @@ storage:
     # root-owned 0700 directory is what it expects.
     - path: /var/lib/qdrant
       mode: 0700
-    # OpenCode project workspace. Scratch space: survives container
+    # OpenCode's workspace and session store; see opencode.service. The
-    # restarts, not a VM taint.
+    # image runs as root.
-    - path: /var/lib/opencode
+    - path: /var/lib/opencode/workspace
-      mode: 0755
-    - path: /etc/opencode
       mode: 0755
+    - path: /var/lib/opencode/data
+      mode: 0700
   files:
-    # LiteLLM proxy settings. Models stay in the database
-    # (STORE_MODEL_IN_DB), so this file carries general_settings only --
-    # those have no environment equivalent and are readable only from a
-    # config file.
-    #
-    # Requests reach :4000 through a Cloudflare tunnel whose connector runs
-    # on 192.168.0.0/24, so every request arrives from that connector's
-    # address. Without use_x_forwarded_for LiteLLM does MCP access control
-    # against that peer, which sits inside the default internal ranges, and
-    # so treats every caller on the public internet as internal -- exposing
-    # MCP servers marked "available_on_public_internet: false".
-    #
-    # mcp_trusted_proxy_ranges is what keeps that safe: X-Forwarded-For is
-    # honoured only when the direct peer is the connector, so a crafted
-    # header from anywhere else cannot forge a client IP. Widen it if the
-    # tunnel ever moves off that subnet, or the header is ignored again.
     # Companion to openwebui-reindex.service above.
     #
     # Note on quoting: templatefile() substitutes on raw text, so shell
@@ -356,6 +372,22 @@ storage:
           echo "Qdrant empty; reindexing knowledge bases"
           curl -fsS -X POST -H "Authorization: Bearer $OWUI_KEY" \
             "$OWUI/api/v1/knowledge/reindex"
+    # LiteLLM proxy settings. Models stay in the database
+    # (STORE_MODEL_IN_DB), so this file carries general_settings only --
+    # those have no environment equivalent and are readable only from a
+    # config file.
+    #
+    # Requests reach :4000 through a Cloudflare tunnel whose connector runs
+    # on 192.168.0.0/24, so every request arrives from that connector's
+    # address. Without use_x_forwarded_for LiteLLM does MCP access control
+    # against that peer, which sits inside the default internal ranges, and
+    # so treats every caller on the public internet as internal -- exposing
+    # MCP servers marked "available_on_public_internet: false".
+    #
+    # mcp_trusted_proxy_ranges is what keeps that safe: X-Forwarded-For is
+    # honoured only when the direct peer is the connector, so a crafted
+    # header from anywhere else cannot forge a client IP. Widen it if the
+    # tunnel ever moves off that subnet, or the header is ignored again.
     - path: /etc/litellm/config.yaml
       mode: 0644
       contents:
@@ -364,35 +396,33 @@ storage:
             use_x_forwarded_for: true
             mcp_trusted_proxy_ranges:
               - 192.168.0.0/24
-    # OpenCode provider config. LiteLLM is an OpenAI-compatible proxy, so
+    # OpenCode's provider config. LiteLLM speaks the OpenAI API, so it is
-    # we use @ai-sdk/openai-compatible rather than a first-party provider.
+    # wired in through @ai-sdk/openai-compatible.
     #
-    # Model keys under "models" must match LiteLLM model_name values exactly
+    # Model keys must match LiteLLM model_name values exactly. Those live in
-    # (they are stored in the LiteLLM DB, not in litellm/config.yaml). Edit
+    # the LiteLLM database, not in a file here, so check GET :4000/v1/models
-    # this list to match what /v1/models returns, then taint the VM -- or
+    # when changing this list. "model" is the default for new sessions.
-    # PATCH http://192.168.7.61:4096/config at runtime. The default model
-    # below is the first key; change "model" when you change the set.
-    #
-    # apiKey is templated from Infisical (LITELLM_MASTER_KEY). Ignition
-    # snippets are plaintext on the node -- same caveat as the other units.
     - path: /etc/opencode/opencode.json
-      mode: 0600
+      mode: 0644
       contents:
         inline: |
           {
             "$schema": "https://opencode.ai/config.json",
-            "model": "litellm/claude-sonnet-4",
+            "model": "litellm/claude-sonnet-5",
             "provider": {
               "litellm": {
                 "npm": "@ai-sdk/openai-compatible",
                 "name": "LiteLLM",
                 "options": {
                   "baseURL": "http://192.168.7.61:4000/v1",
-                  "apiKey": "${litellm_master_key}"
+                  "apiKey": "{env:LITELLM_API_KEY}"
                 },
                 "models": {
-                  "claude-sonnet-4": {
+                  "claude-sonnet-5": {
-                    "name": "Claude Sonnet 4"
+                    "name": "Claude Sonnet 5"
+                  },
+                  "qwen3:4b-instruct": {
+                    "name": "Qwen3 4B Instruct"
                   }
                 }
               }

butane/cisa_kev_rss.yaml

@@ -1,52 +0,0 @@
-variant: flatcar
-version: 1.0.0
-
-# CISA KEV RSS feed. Turns the CISA Known Exploited Vulnerabilities catalog
-# into an RSS 2.0 feed (https://github.com/xsaveopt/cisa-kev-rss).
-#
-# Feed:  http://192.168.7.64:3000/rss
-# Health: http://192.168.7.64:3000/health
-#
-# Stateless: the container refreshes from CISA every ten minutes and keeps
-# nothing on disk, so it is safe to rebuild at any time.
-
-systemd:
-  units:
-    - name: cisa-kev-rss.service
-      enabled: true
-      contents: |
-        [Unit]
-        Description=CISA KEV RSS feed
-        After=docker.service
-        Requires=docker.service
-        [Service]
-        TimeoutStartSec=0
-        ExecStartPre=/usr/bin/docker pull ghcr.io/xsaveopt/cisa-kev-rss:latest
-        ExecStartPre=-/usr/bin/docker rm --force cisa-kev-rss
-        ExecStart=/usr/bin/docker run --rm --name cisa-kev-rss \
-          -p 3000:3000 \
-          ghcr.io/xsaveopt/cisa-kev-rss:latest
-        ExecStop=/usr/bin/docker stop cisa-kev-rss
-        Restart=always
-        RestartSec=5s
-        [Install]
-        WantedBy=multi-user.target
-    - name: dozzle-agent.service
-      enabled: true
-      contents: |
-        [Unit]
-        Description=Dozzle Agent
-        After=docker.service
-        Requires=docker.service
-        [Service]
-        TimeoutStartSec=0
-        ExecStartPre=-/usr/bin/docker rm --force dozzle-agent
-        ExecStart=/usr/bin/docker run --name dozzle-agent \
-          -v /var/run/docker.sock:/var/run/docker.sock \
-          -p 7007:7007 \
-          amir20/dozzle:v11.1.1 agent
-        ExecStop=/usr/bin/docker stop dozzle-agent
-        Restart=always
-        RestartSec=5s
-        [Install]
-        WantedBy=multi-user.target

butane/monitor.yaml.tftpl

@@ -13,6 +13,8 @@ version: 1.0.0
 # VMs (see the dozzle-agent units in the other butane configs) and, via the
 # local socket, from this host's own containers. Reachable on :8080.
 #
+# Also runs Alerta on :8082; see alerta.service below.
+#
 # No state is kept on disk: /data is a tmpfs, so the container is fully
 # disposable and there is nothing to back up. That is also why simple auth
 # (users.yml) is not used -- it would need a persistent /data.
@@ -20,10 +22,11 @@ version: 1.0.0
 # Hub <-> agent traffic is TLS, using the certificate pair baked into the
 # dozzle image. Every hub and agent on the same image version shares it, so
 # nothing needs mounting here -- which is also why the agents must stay pinned
-# to the same tag as the hub. To move to a custom pair later, generate one
+# to the same tag as the hub (DOZZLE_IMAGE in versions.env). To move to a
-# with (writes into $PWD):
+# custom pair later, generate one with (writes into $PWD, using the tag from
+# DOZZLE_IMAGE):
 #
-#   docker run --rm -v "$PWD":/out amir20/dozzle:v11.1.1 generate-certs \
+#   docker run --rm -v "$PWD":/out amir20/dozzle:<tag> generate-certs \
 #     --cert-out /out/dozzle_cert.pem --key-out /out/dozzle_key.pem
 #
 # ...then mount it at /dozzle_cert.pem and /dozzle_key.pem (read-only) on the
@@ -37,9 +40,12 @@ systemd:
       contents: |
         [Unit]
         Description=Dozzle log viewer hub
-        After=docker.service
+        After=docker.service versions-fetch.service
         Requires=docker.service
+        Wants=versions-fetch.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         # Kept as unit environment rather than inline -e values so that
         # wait-for-agents.sh can read the agent list from the same place the
@@ -63,7 +69,7 @@ systemd:
         Environment="DOZZLE_AUTH_OIDC_SCOPES=dozzle_roles,picture"
         Environment="DOZZLE_AUTH_OIDC_ROLES_CLAIM=dozzle_roles"
         # Pulled up front so a slow first pull is not mistaken for a hung unit.
-        ExecStartPre=/usr/bin/docker pull amir20/dozzle:v11.1.1
+        ExecStartPre=/usr/bin/docker pull $${DOZZLE_IMAGE}
         # Leading "-" so a missing container is not a failure.
         ExecStartPre=-/usr/bin/docker rm --force dozzle
         # If the hub starts before the agents are listening, the failed initial
@@ -85,7 +91,7 @@ systemd:
           -e DOZZLE_AUTH_OIDC_NAME \
           -e DOZZLE_AUTH_OIDC_SCOPES \
           -e DOZZLE_AUTH_OIDC_ROLES_CLAIM \
-          amir20/dozzle:v11.1.1
+          $${DOZZLE_IMAGE}
         ExecStop=/usr/bin/docker stop dozzle
         Restart=always
         RestartSec=5s
@@ -108,15 +114,18 @@ systemd:
       contents: |
         [Unit]
         Description=Dozzle MCP endpoint
-        After=docker.service
+        After=docker.service versions-fetch.service
         Requires=docker.service
+        Wants=versions-fetch.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         # Same agents as the hub, same file, so the two cannot drift.
         EnvironmentFile=/etc/dozzle/agents.env
         Environment="DOZZLE_HOSTNAME=Oswald Logs (MCP)"
         Environment="DOZZLE_ENABLE_MCP=true"
-        ExecStartPre=/usr/bin/docker pull amir20/dozzle:v11.1.1
+        ExecStartPre=/usr/bin/docker pull $${DOZZLE_IMAGE}
         ExecStartPre=-/usr/bin/docker rm --force dozzle-mcp
         ExecStartPre=-/usr/bin/timeout 180 /opt/dozzle/wait-for-agents.sh
         ExecStart=/usr/bin/docker run --rm --name dozzle-mcp \
@@ -126,12 +135,73 @@ systemd:
           -e DOZZLE_HOSTNAME \
           -e DOZZLE_REMOTE_AGENT \
           -e DOZZLE_ENABLE_MCP \
-          amir20/dozzle:v11.1.1
+          $${DOZZLE_IMAGE}
         ExecStop=/usr/bin/docker stop dozzle-mcp
         Restart=always
         RestartSec=5s
         [Install]
         WantedBy=multi-user.target
+    # Alerta alert console (API + web UI) on :8082.
+    #
+    # Prerequisites, all of which must exist before the VM is rebuilt:
+    #   - an "alerta" Postgres database and role; the schema is created on
+    #     first start
+    #   - an OIDC client for Alerta in Authelia, with the Alerta URL as a
+    #     redirect URI and token_endpoint_auth_method client_secret_basic
+    #   - ALERTA_DATABASE_URL, ALERTA_SECRET_KEY, ALERTA_ADMIN_PASSWORD,
+    #     ALERTA_OIDC_CLIENT_ID and ALERTA_OIDC_CLIENT_SECRET in
+    #     Infisical (see secrets.tmpl). Every Butane file is rendered on every
+    #     plan, so until they exist no plan succeeds -- not just this VM's.
+    #     Generate the key with `openssl rand -hex 32`, and keep "%" out of
+    #     the database URL: systemd reads it as a specifier
+    #
+    # Alerts, users and API keys all live in Postgres, so a taint loses
+    # nothing. SECRET_KEY signs login tokens; without a fixed one the image
+    # generates a new one per start and every session is dropped.
+    #
+    # ADMIN_PASSWORD is used only when the admin user does not exist yet.
+    # Changing it in Infisical afterwards has no effect: change it in the UI.
+    # Web logins go through Authelia; the basic-auth admin user is kept for
+    # API access. Self sign-up is off, so further users are added by the admin.
+    #
+    # --log-driver none keeps this container out of Dozzle. The image's
+    # entrypoint dumps its whole environment and prints the admin API key at
+    # every start, and dozzle-mcp above serves logs without auth. Output
+    # still reaches the journal (journalctl -u alerta) through the attached
+    # docker run.
+    - name: alerta.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Alerta alert console
+        After=docker.service versions-fetch.service
+        Requires=docker.service
+        Wants=versions-fetch.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        ExecStartPre=/usr/bin/docker pull $${ALERTA_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force alerta
+        ExecStart=/usr/bin/docker run --rm --name alerta \
+          --log-driver none \
+          -e DATABASE_URL=${alerta_database_url} \
+          -e SECRET_KEY=${alerta_secret_key} \
+          -e AUTH_REQUIRED=True \
+          -e AUTH_PROVIDER=openid \
+          -e OIDC_ISSUER_URL=https://auth.atlashorizon.net \
+          -e OAUTH2_CLIENT_ID=${alerta_oidc_client_id} \
+          -e OAUTH2_CLIENT_SECRET=${alerta_oidc_client_secret} \
+          -e SIGNUP_ENABLED=True \
+          -e ADMIN_USERS=admin \
+          -e ADMIN_PASSWORD=${alerta_admin_password} \
+          -p 8082:8080 \
+          $${ALERTA_IMAGE}
+        ExecStop=/usr/bin/docker stop alerta
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
 
 storage:
   files:
@@ -142,7 +212,7 @@ storage:
       mode: 0644
       contents:
         inline: |
-          DOZZLE_REMOTE_AGENT=192.168.7.60:7007|oswald-runner|Goose,192.168.7.61:7007|ai|Goose,192.168.7.62:7007|oswl-connector|Goose,192.168.7.64:7007|cisa-kev-rss|Goose,192.168.7.4:7007|PortaNAS|Portanas
+          DOZZLE_REMOTE_AGENT=192.168.7.60:7007|oswald-runner|Goose,192.168.7.61:7007|ai|Goose,192.168.7.62:7007|oswl-connector|Goose,192.168.20.74:7007|Argo|Truenas,192.168.20.4:7007|Atalantia|Truenas,192.168.7.4:7007|Goldie|Truenas,192.168.20.217:7007|Laika|CallieCorner
     # Blocks until every agent named in DOZZLE_REMOTE_AGENT accepts a TCP
     # connection. The list is read from the environment, so adding or removing
     # an agent above needs no change here.

butane/oswald_runner.yaml.tftpl

@@ -25,9 +25,12 @@ systemd:
       contents: |
         [Unit]
         Description=Gitea Actions runner
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force oswald_runner
         ExecStart=/usr/bin/docker run --name oswald_runner \
@@ -36,7 +39,7 @@ systemd:
           -e GITEA_RUNNER_NAME=oswald_runner \
           -v /var/lib/oswald_runner:/data \
           -v /var/run/docker.sock:/var/run/docker.sock \
-          docker.io/gitea/runner:3
+          $${GITEA_RUNNER_IMAGE}
         ExecStop=/usr/bin/docker stop oswald_runner
         Restart=always
         RestartSec=5s
@@ -61,11 +64,14 @@ systemd:
       contents: |
         [Unit]
         Description=n8n workflow automation
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
-        ExecStartPre=/usr/bin/docker pull docker.n8n.io/n8nio/n8n:2.40.5
+        ExecStartPre=/usr/bin/docker pull $${N8N_IMAGE}
         ExecStartPre=-/usr/bin/docker rm --force n8n
         # /var/lib/n8n holds only what is left in /home/node/.n8n once the
         # database and key are external: installed community nodes and
@@ -87,28 +93,203 @@ systemd:
           -e N8N_PROTOCOL=https \
           -e WEBHOOK_URL=https://n8n.atlashorizon.net/ \
           -e N8N_PROXY_HOPS=1 \
+          -e N8N_DIAGNOSTICS_ENABLED=false \
           -v /var/lib/n8n:/home/node/.n8n \
           -p 5678:5678 \
-          docker.n8n.io/n8nio/n8n:2.40.5
+          $${N8N_IMAGE}
         ExecStop=/usr/bin/docker stop n8n
         Restart=always
         RestartSec=5s
         [Install]
         WantedBy=multi-user.target
+    # CISA KEV RSS feed. Turns the CISA Known Exploited Vulnerabilities
+    # catalog into an RSS 2.0 feed (https://github.com/xsaveopt/cisa-kev-rss).
+    #
+    # Feed:   http://192.168.7.60:3000/rss
+    # Health: http://192.168.7.60:3000/health
+    #
+    # Stateless: the container refreshes from CISA every ten minutes and keeps
+    # nothing on disk. Moved here from its own VM (cisa-kev-rss, 192.168.7.64).
+    - name: cisa-kev-rss.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=CISA KEV RSS feed
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
+        Requires=docker.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        ExecStartPre=/usr/bin/docker pull $${CISA_KEV_RSS_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force cisa-kev-rss
+        ExecStart=/usr/bin/docker run --rm --name cisa-kev-rss \
+          -p 3000:3000 \
+          $${CISA_KEV_RSS_IMAGE}
+        ExecStop=/usr/bin/docker stop cisa-kev-rss
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # Domain Watchdog: RDAP-based domain expiry/change monitoring
+    # (https://github.com/maelgangloff/domain-watchdog), served at
+    # domainwatchdog.atlashorizon.net. Three containers on a private docker
+    # network: the web app on :8083, a messenger worker, and Valkey as the
+    # job queue and lock store.
+    #
+    # Prerequisites, all of which must exist before the VM is rebuilt:
+    #   - a "domainwatchdog" Postgres database and role on
+    #     argo.lan.atlashorizon.net; the schema is migrated on every start
+    #   - an OIDC client in Authelia with redirect URI
+    #     https://domainwatchdog.atlashorizon.net/login/oauth/check, scopes
+    #     openid/email/profile, and token_endpoint_auth_method
+    #     client_secret_post (the app sends the secret in the POST body)
+    #   - DOMAIN_WATCHDOG_DB_PASSWORD, DOMAIN_WATCHDOG_APP_SECRET,
+    #     DOMAIN_WATCHDOG_OIDC_CLIENT_ID and DOMAIN_WATCHDOG_OIDC_CLIENT_SECRET
+    #     in Infisical (see secrets.tmpl). Every Butane file is rendered on
+    #     every plan, so until they exist no plan succeeds. Generate the app
+    #     secret with `openssl rand -hex 32`; keep the DB password URL-safe
+    #     and free of "%" (it is spliced into DATABASE_URL, and systemd reads
+    #     "%" as a specifier)
+    #   - domainwatchdog.atlashorizon.net terminating TLS and proxying to this
+    #     host's :8083 with X-Forwarded-Proto/Host (not managed here)
+    #
+    # Users, watchlists and domain history all live in Postgres, so a taint
+    # loses nothing that matters. What does not survive it:
+    #   - queued RDAP jobs in Valkey; the scheduler re-queues them
+    #   - the JWT signing keys in /var/lib/domain-watchdog/jwt, which the
+    #     entrypoint regenerates when missing, so everyone logs in again.
+    #     They are on disk rather than in the container only so that a
+    #     plain restart does not do the same.
+    #
+    # Login is Authelia-only: registration is off, and the first OIDC login
+    # for an email creates that user, so Authelia's access rules for the
+    # client decide who gets in.
+    - name: domain-watchdog-valkey.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Domain Watchdog Valkey
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
+        Requires=docker.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        ExecStartPre=/usr/bin/docker pull $${DOMAIN_WATCHDOG_VALKEY_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force domain-watchdog-valkey
+        # Every Domain Watchdog unit tries to create the network, so start
+        # order does not matter; "-" because all but the first will fail.
+        ExecStartPre=-/usr/bin/docker network create domain-watchdog
+        # No volume and no port: the queue is disposable and only the other
+        # two containers reach it, by name, over the domain-watchdog network.
+        ExecStart=/usr/bin/docker run --rm --name domain-watchdog-valkey \
+          --network domain-watchdog \
+          --network-alias valkey \
+          $${DOMAIN_WATCHDOG_VALKEY_IMAGE}
+        ExecStop=/usr/bin/docker stop domain-watchdog-valkey
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    - name: domain-watchdog.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Domain Watchdog web app
+        After=docker.service domain-watchdog-valkey.service versions-fetch.service
+        Requires=docker.service
+        Wants=domain-watchdog-valkey.service versions-fetch.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        # Secrets are unit environment rather than lines in app.env, which
+        # is world-readable; docker run forwards them by bare name below.
+        Environment="APP_SECRET=${domain_watchdog_app_secret}"
+        Environment="DATABASE_URL=postgresql://domainwatchdog:${domain_watchdog_db_password}@argo.lan.atlashorizon.net:5432/domainwatchdog?serverVersion=16&charset=utf8"
+        Environment="OAUTH_CLIENT_ID=${domain_watchdog_oidc_client_id}"
+        Environment="OAUTH_CLIENT_SECRET=${domain_watchdog_oidc_client_secret}"
+        ExecStartPre=/usr/bin/docker pull $${DOMAIN_WATCHDOG_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force domain-watchdog
+        ExecStartPre=-/usr/bin/docker network create domain-watchdog
+        # The entrypoint runs DB migrations and generates the JWT keypair
+        # into /app/config/jwt if it is missing, then starts FrankenPHP on
+        # :80 (plain HTTP: SERVER_NAME=:80 in app.env, TLS is the proxy's).
+        ExecStart=/usr/bin/docker run --rm --name domain-watchdog \
+          --network domain-watchdog \
+          --env-file /etc/domain-watchdog/app.env \
+          -e APP_SECRET \
+          -e DATABASE_URL \
+          -e OAUTH_CLIENT_ID \
+          -e OAUTH_CLIENT_SECRET \
+          -v /var/lib/domain-watchdog/jwt:/app/config/jwt \
+          -v /etc/domain-watchdog/content:/app/public/content:ro \
+          -p 8083:80 \
+          $${DOMAIN_WATCHDOG_IMAGE}
+        ExecStop=/usr/bin/docker stop domain-watchdog
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # Runs the queued RDAP lookups and notifications. Same image, so the
+    # entrypoint also runs migrations here; ordering after the web app keeps
+    # the two from racing on a fresh database. It exits every hour
+    # (--time-limit, as upstream recommends against PHP memory creep) and
+    # Restart=always brings it straight back.
+    - name: domain-watchdog-worker.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Domain Watchdog worker
+        After=docker.service domain-watchdog-valkey.service domain-watchdog.service versions-fetch.service
+        Requires=docker.service
+        Wants=domain-watchdog-valkey.service versions-fetch.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        Environment="APP_SECRET=${domain_watchdog_app_secret}"
+        Environment="DATABASE_URL=postgresql://domainwatchdog:${domain_watchdog_db_password}@argo.lan.atlashorizon.net:5432/domainwatchdog?serverVersion=16&charset=utf8"
+        ExecStartPre=/usr/bin/docker pull $${DOMAIN_WATCHDOG_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force domain-watchdog-worker
+        ExecStartPre=-/usr/bin/docker network create domain-watchdog
+        # --no-healthcheck: the image's HEALTHCHECK probes FrankenPHP's
+        # metrics on :2019, which only the web app runs, so the worker would
+        # always show unhealthy (upstream's compose disables it the same way).
+        ExecStart=/usr/bin/docker run --rm --name domain-watchdog-worker \
+          --no-healthcheck \
+          --network domain-watchdog \
+          --env-file /etc/domain-watchdog/app.env \
+          -e APP_SECRET \
+          -e DATABASE_URL \
+          -e MESSENGER_CONSUMER_NAME=worker \
+          $${DOMAIN_WATCHDOG_IMAGE} \
+          php /app/bin/console messenger:consume --all --time-limit=3600 -vv
+        ExecStop=/usr/bin/docker stop domain-watchdog-worker
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
     - name: dozzle-agent.service
       enabled: true
       contents: |
         [Unit]
         Description=Dozzle Agent
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force dozzle-agent
         ExecStart=/usr/bin/docker run --name dozzle-agent \
           -v /var/run/docker.sock:/var/run/docker.sock \
           -p 7007:7007 \
-          amir20/dozzle:v11.1.1 agent
+          $${DOZZLE_IMAGE} agent
         ExecStop=/usr/bin/docker stop dozzle-agent
         Restart=always
         RestartSec=5s
@@ -127,3 +308,77 @@ storage:
         id: 1000
       group:
         id: 1000
+    # Domain Watchdog's JWT keypair; the container runs as root.
+    - path: /var/lib/domain-watchdog/jwt
+      mode: 0700
+  files:
+    # Non-secret Domain Watchdog settings, shared by the web app and worker
+    # units (docker --env-file format: no quoting, values taken literally).
+    - path: /etc/domain-watchdog/app.env
+      mode: 0644
+      contents:
+        inline: |
+          APP_ENV=prod
+          SERVER_NAME=:80
+          HTTP_SECURE_COOKIE=true
+          SYMFONY_TRUSTED_PROXIES=private_ranges
+          SYMFONY_TRUSTED_HEADERS=x-forwarded-for,x-forwarded-host,x-forwarded-proto,x-forwarded-port
+          LOCK_DSN=redis://valkey:6379
+          MESSENGER_ASYNC_TRANSPORT_DSN=redis://valkey:6379/messages
+          MESSENGER_RDAP_LOW_TRANSPORT_DSN=redis://valkey:6379/messages-rdap-low
+          MESSENGER_RDAP_HIGH_TRANSPORT_DSN=redis://valkey:6379/messages-rdap-high
+          REGISTRATION_ENABLED=false
+          REGISTRATION_VERIFY_EMAIL=false
+          OAUTH_AUTHORIZATION_URL=https://auth.atlashorizon.net/api/oidc/authorization
+          OAUTH_TOKEN_URL=https://auth.atlashorizon.net/api/oidc/token
+          OAUTH_USERINFO_URL=https://auth.atlashorizon.net/api/oidc/userinfo
+          OAUTH_SCOPE=openid email profile
+          SSO_AUTO_REDIRECT=true
+          MAILER_DSN=null://null
+    # Markdown pages the frontend fetches from /content/<name>; a missing one
+    # renders as "Please create the /public/content/<name> file." Mounted
+    # read-only into the web app. ads.md is shown on the login page and is
+    # left empty on purpose (an empty file renders as nothing).
+    - path: /etc/domain-watchdog/content/home.md
+      mode: 0644
+      contents:
+        inline: |
+          # Domain Watchdog
+
+          Tracks our domains through RDAP and alerts on expiry, transfers and
+          other registry changes.
+
+          Add domains under **Tracking > Watchlists**, and attach a
+          notification connector to a watchlist to be alerted.
+    - path: /etc/domain-watchdog/content/faq.md
+      mode: 0644
+      contents:
+        inline: |
+          # FAQ
+
+          **Who can log in?** Anyone allowed through Authelia for this app.
+          Your account is created on first login.
+
+          **How often are domains checked?** The worker refreshes watched
+          domains from RDAP on a schedule; domains close to expiry are
+          checked more often.
+    - path: /etc/domain-watchdog/content/tos.md
+      mode: 0644
+      contents:
+        inline: |
+          # Terms of use
+
+          Private instance for Atlas Horizon use only.
+    - path: /etc/domain-watchdog/content/privacy.md
+      mode: 0644
+      contents:
+        inline: |
+          # Privacy
+
+          Stores your email address (from Authelia) and your watchlists.
+          Nothing is shared with third parties; RDAP lookups go directly to
+          the registries.
+    - path: /etc/domain-watchdog/content/ads.md
+      mode: 0644
+      contents:
+        inline: ""

butane/oswl_connector.yaml.tftpl

@@ -20,10 +20,12 @@ systemd:
       contents: |
         [Unit]
         Description=Tailscale node (headscale subnet router + exit node)
-        After=docker.service network-online.target
+        After=docker.service network-online.target versions-fetch.service
         Requires=docker.service
-        Wants=network-online.target
+        Wants=network-online.target versions-fetch.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         # None of these are loaded by default on a stock Flatcar boot, and the
         # container cannot load them itself: it has NET_ADMIN (enough to write
@@ -49,7 +51,7 @@ systemd:
           -e TS_ACCEPT_DNS=false \
           -e "TS_EXTRA_ARGS=--login-server=https://hs.atlashorizon.net --advertise-exit-node" \
           -e TS_USERSPACE=false \
-          docker.io/tailscale/tailscale:stable
+          $${TAILSCALE_IMAGE}
         ExecStop=/usr/bin/docker stop tailscaled
         Restart=always
         RestartSec=5s
@@ -60,15 +62,18 @@ systemd:
       contents: |
         [Unit]
         Description=Dozzle Agent
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force dozzle-agent
         ExecStart=/usr/bin/docker run --name dozzle-agent \
           -v /var/run/docker.sock:/var/run/docker.sock \
           -p 7007:7007 \
-          amir20/dozzle:v11.1.1 agent
+          $${DOZZLE_IMAGE} agent
         ExecStop=/usr/bin/docker stop dozzle-agent
         Restart=always
         RestartSec=5s

butane/versions-sync.yaml

@@ -0,0 +1,113 @@
+variant: flatcar
+version: 1.0.0
+
+# Shared by every Flatcar VM: main.tf passes this file as each VM's inline
+# butane snippet. Loaded with file(), not templatefile(), so it is not a
+# template -- ${...} below reaches the VM as written.
+#
+# Keeps /etc/versions.env in sync with versions.env at the root of
+# AtlasHorizon/compose, which holds the image tag of every container on the
+# Flatcar VMs. A version bump merged to the branch below reaches each VM
+# within ~10 minutes with no plan or rebuild; see fetch-versions.sh for how
+# the affected units are found and restarted.
+#
+# To opt a container unit in, in its butane file:
+#   [Unit]     After=... versions-fetch.service
+#              Wants=versions-fetch.service
+#   [Service]  EnvironmentFile=/etc/versions.env
+# and use the variable, in braces, wherever the image goes (the docker pull
+# and docker run arguments). In a .tftpl file the braces need a doubled
+# dollar sign so templatefile() leaves them alone.
+
+systemd:
+  units:
+    # Not enabled itself: the container units pull it in at boot (Wants= and
+    # After=), so they start on the freshest copy, and the timer re-runs it.
+    # A failed fetch keeps the last good copy, so a Gitea outage only delays
+    # updates -- except on a brand-new VM, whose units cannot start until the
+    # first fetch succeeds. Restart= keeps retrying that first fetch.
+    - name: versions-fetch.service
+      contents: |
+        [Unit]
+        Description=Fetch image versions from AtlasHorizon/compose
+        Wants=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        # Override with a drop-in to test unreleased versions on one VM.
+        Environment=VERSIONS_BRANCH=main
+        ExecStart=/opt/versions/fetch-versions.sh
+        Restart=on-failure
+        RestartSec=30s
+    - name: versions-fetch.timer
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Periodically re-fetch image versions
+        [Timer]
+        OnBootSec=10min
+        OnUnitActiveSec=10min
+        RandomizedDelaySec=1min
+        [Install]
+        WantedBy=timers.target
+
+storage:
+  files:
+    - path: /opt/versions/fetch-versions.sh
+      mode: 0755
+      contents:
+        inline: |
+          #!/usr/bin/bash
+          # Fetches versions.env and restarts the units that use a variable
+          # whose value changed. A unit "uses" a variable when its unit file
+          # contains it in braces (the docker pull/run image arguments), so
+          # adding a container needs no change here.
+          set -euo pipefail
+
+          url="https://git.atlashorizon.net/AtlasHorizon/compose/raw/branch/$VERSIONS_BRANCH/versions.env"
+          dest=/etc/versions.env
+
+          tmp=$(mktemp /etc/.versions.env.XXXXXX)
+          trap 'rm -f "$tmp"' EXIT
+
+          curl -fsS --max-time 30 -o "$tmp" "$url"
+
+          # Only comments, blank lines and VAR=value lines are accepted, and
+          # at least one *_IMAGE line must be present, so an error page or a
+          # truncated download never replaces the last good copy.
+          if grep -qvE '^(#.*|[A-Z0-9_]+=[^[:space:]]+|)$' "$tmp" ||
+             ! grep -qE '^[A-Z0-9_]+_IMAGE=' "$tmp"; then
+            echo "refusing malformed versions.env from $url" >&2
+            exit 1
+          fi
+
+          if [ -f "$dest" ] && cmp -s "$tmp" "$dest"; then
+            exit 0
+          fi
+
+          # With no previous copy this is a brand-new VM: its units are waiting
+          # on (or already failed without) this file, so start them outright.
+          # Otherwise only units that are running get restarted, so a unit
+          # stopped by hand stays stopped.
+          if [ -f "$dest" ]; then
+            action=try-restart
+            changed=$(diff <(grep -E '^[A-Z0-9_]+=' "$dest" | sort) \
+                           <(grep -E '^[A-Z0-9_]+=' "$tmp" | sort) |
+                      sed -nE 's/^[<>] ([A-Z0-9_]+)=.*/\1/p' | sort -u || true)
+          else
+            action=restart
+            changed=$(sed -nE 's/^([A-Z0-9_]+)=.*/\1/p' "$tmp")
+          fi
+
+          chmod 0644 "$tmp"
+          mv -f "$tmp" "$dest"
+
+          for var in $changed; do
+            echo "$var changed"
+            pattern=$(printf '${%s}' "$var")
+            # --no-block: the units are ordered after this one, so waiting on
+            # their jobs from inside it would deadlock at boot.
+            for unit in $(grep -lF "$pattern" /etc/systemd/system/*.service || true); do
+              systemctl --no-block "$action" "$(basename "$unit")"
+            done
+          done

main.tf

@@ -36,6 +36,20 @@ locals {
 # provider alias in providers.tf and a module block here. Everything else
 # (node contents, common config) lives in the parent repo's terraform.tfvars.
 
+# butane/versions-sync.yaml goes onto every Flatcar VM through its inline
+# butane field (the module merges it with butane_file). Inline snippets are
+# not templated, so the file is read as-is. No VM sets butane in tfvars; one
+# that did would replace this snippet, not add to it, and lose version sync.
+locals {
+  flatcar_common_butane = file("${path.module}/butane/versions-sync.yaml")
+  flatcar_with_common = {
+    for hv in ["proxmox", "goose"] : hv => {
+      for name, vm in try(var.nodes[hv].flatcar, {}) :
+      name => merge(vm, { butane = coalesce(vm.butane, local.flatcar_common_butane) })
+    }
+  }
+}
+
 module "proxmox" {
   source = "./opentofu/modules/proxmox_node"
 
@@ -43,7 +57,7 @@ module "proxmox" {
   storage              = var.hypervisors["proxmox"].storage
   lxc                  = try(var.nodes["proxmox"].lxc, {})
   machines             = try(var.nodes["proxmox"].machines, {})
-  flatcar              = try(var.nodes["proxmox"].flatcar, {})
+  flatcar              = local.flatcar_with_common["proxmox"]
   search_domain        = local.node_common.search_domain
   dns_nameservers      = local.node_common.dns_nameservers
   ssh_key              = local.node_common.ssh_key
@@ -68,7 +82,7 @@ module "goose" {
   storage              = var.hypervisors["goose"].storage
   lxc                  = try(var.nodes["goose"].lxc, {})
   machines             = try(var.nodes["goose"].machines, {})
-  flatcar              = try(var.nodes["goose"].flatcar, {})
+  flatcar              = local.flatcar_with_common["goose"]
   search_domain        = local.node_common.search_domain
   dns_nameservers      = local.node_common.dns_nameservers
   ssh_key              = local.node_common.ssh_key

secrets.tmpl

@@ -29,6 +29,15 @@
   {{- if eq $secret.Key "N8N_ENCRYPTION_KEY" }}"n8n_encryption_key": "{{$secret.Value}}",{{ end }}
   {{- if eq $secret.Key "DOZZLE_OIDC_CLIENT_ID" }}"dozzle_oidc_client_id": "{{$secret.Value}}",{{ end }}
   {{- if eq $secret.Key "DOZZLE_OIDC_CLIENT_SECRET" }}"dozzle_oidc_client_secret": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_DATABASE_URL" }}"alerta_database_url": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_SECRET_KEY" }}"alerta_secret_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_ADMIN_PASSWORD" }}"alerta_admin_password": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_OIDC_CLIENT_ID" }}"alerta_oidc_client_id": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_OIDC_CLIENT_SECRET" }}"alerta_oidc_client_secret": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "DOMAIN_WATCHDOG_DB_PASSWORD" }}"domain_watchdog_db_password": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "DOMAIN_WATCHDOG_APP_SECRET" }}"domain_watchdog_app_secret": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "DOMAIN_WATCHDOG_OIDC_CLIENT_ID" }}"domain_watchdog_oidc_client_id": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "DOMAIN_WATCHDOG_OIDC_CLIENT_SECRET" }}"domain_watchdog_oidc_client_secret": "{{$secret.Value}}",{{ end }}
   {{- end }}
   "_sentinel": "1"
 }

terraform.tfvars

@@ -203,14 +203,10 @@ nodes = {
         vmid             = 210
         ip               = "192.168.7.60"
         vlan             = 7
-        memory           = 4096
+        memory           = 6144
         cores            = 4
         disk_size        = 32
         qemu_guest_agent = false
-        # Rendered with templatefile(): ${gitea_runner_token} comes from
-        # Infisical (GITEA_RUNNER_TOKEN in secrets.tmpl), so the token is
-        # never committed. Also runs n8n on :5678 (N8N_* secrets, Postgres
-        # on argo); see the prerequisites in the Butane file.
         butane_file = "butane/oswald_runner.yaml.tftpl"
       }
       "ai" = {
@@ -218,16 +214,12 @@ nodes = {
         vmid             = 211
         ip               = "192.168.7.61"
         vlan             = 7
-        memory           = 4096
+        memory           = 8192
-        cores            = 2
+        cores            = 4
         disk_size        = 32
         qemu_guest_agent = false
-        # OPENCODE_SERVER_PASSWORD in Infisical; see butane/ai.yaml.tftpl.
         butane_file = "butane/ai.yaml.tftpl"
       }
-      # Headscale subnet router / exit node. Registers as an ephemeral node,
-      # so the preauth key must be --ephemeral --reusable; the routes and the
-      # exit node still need approving on the headscale side once it appears.
       "oswl-connector" = {
         hostname         = "oswl-connector"
         vmid             = 212
@@ -239,33 +231,17 @@ nodes = {
         qemu_guest_agent = false
         butane_file      = "butane/oswl_connector.yaml.tftpl"
       }
-      # Dozzle log viewer hub. Pulls container logs from the dozzle agents on
-      # the other Flatcar VMs (port 7007) plus its own local docker socket.
-      # Keeps no state on disk, so it is safe to rebuild at any time.
       "monitor" = {
         hostname         = "monitor"
         vmid             = 213
         ip               = "192.168.7.63"
         vlan             = 7
-        memory           = 2048
+        memory           = 4096
         cores            = 2
         disk_size        = 32
         qemu_guest_agent = false
         butane_file      = "butane/monitor.yaml.tftpl"
       }
-      # CISA Known Exploited Vulnerabilities catalog as an RSS feed.
-      # Stateless container; feed at :3000/rss.
-      "cisa-kev-rss" = {
-        hostname         = "cisa-kev-rss"
-        vmid             = 214
-        ip               = "192.168.7.64"
-        vlan             = 7
-        memory           = 1024
-        cores            = 1
-        disk_size        = 16
-        qemu_guest_agent = false
-        butane_file      = "butane/cisa_kev_rss.yaml"
-      }
     }
     machines = {
       "grist" = {
@@ -280,7 +256,7 @@ nodes = {
         agent      = 1
         memory     = 4096
         disk_size  = "32"
-        balloon    = 1
+        balloon    = 1024
         cpu_type   = "kvm64"
         cores      = 4
         sockets    = 1