Changed how all docker images have their version set to use the compose repo

Add OpenCode to goose ai Flatcar for n8n

ansible/inventories/proxmox.yml

@@ -0,0 +1,10 @@
+proxmox:
+  hosts:
+    pve-goose:
+      ansible_host: 192.168.7.240
+    pve-duck:
+      ansible_host: 192.168.7.241
+
+  vars:
+    ansible_user: "ansible"
+    ansible_ssh_private_key_file: "/home/nick/.ssh/infra_ansible_key"

ansible/plays/machine_setup/domain-watchdog.yml

@@ -24,21 +24,28 @@
 
     - name: Deploy Domain Watchdog
       vars:
+        # Keep this path: compose prefixes named volumes (database_data etc.)
+        # with the directory name, so moving it orphans the existing data.
         domain_watchdog_dir: /opt/domain-watchdog
+        # Branch of AtlasHorizon/compose to deploy from; override with
+        # `-e compose_branch=<branch>` to test unmerged changes.
+        compose_branch: main
       block:
-        - name: Clone domain-watchdog repo
-          ansible.builtin.git:
-            repo: https://github.com/maelgangloff/domain-watchdog.git
-            dest: "{{ domain_watchdog_dir }}"
-            version: master
-            update: yes
-
         - name: Create public/content directory
           ansible.builtin.file:
             path: "{{ domain_watchdog_dir }}/public/content"
             state: directory
             mode: "0755"
 
+        # force: true so changes in the compose repo are picked up on reruns;
+        # it only reports changed when the content differs.
+        - name: Download docker-compose.yml
+          ansible.builtin.get_url:
+            url: "https://git.atlashorizon.net/AtlasHorizon/compose/raw/branch/{{ compose_branch }}/domain-watchdog/docker-compose.yml"
+            dest: "{{ domain_watchdog_dir }}/docker-compose.yml"
+            mode: "0644"
+            force: true
+
         # Pulled from Infisical, not generated/persisted to disk. Run this
         # playbook as `infisical run -- ansible-playbook -i inventory ...`
         # with DOMAIN_WATCHDOG_POSTGRES_PASSWORD, DOMAIN_WATCHDOG_APP_SECRET,

ansible/plays/machine_setup/grist.yml

@@ -27,6 +27,9 @@
         grist_dir: /opt/grist
         grist_hostname: grist.atlashorizon.net
         grist_oidc_issuer_host: auth.atlashorizon.net
+        # Branch of AtlasHorizon/compose to deploy from; override with
+        # `-e compose_branch=<branch>` to test unmerged changes.
+        compose_branch: main
       block:
         - name: Create persist directory
           ansible.builtin.file:
@@ -34,11 +37,14 @@
             state: directory
             mode: "0755"
 
-        - name: Deploy docker-compose.yml
-          ansible.builtin.copy:
-            src: templates/grist-docker-compose.yml
+        # force: true so changes in the compose repo are picked up on reruns;
+        # it only reports changed when the content differs.
+        - name: Download docker-compose.yml
+          ansible.builtin.get_url:
+            url: "https://git.atlashorizon.net/AtlasHorizon/compose/raw/branch/{{ compose_branch }}/grist/docker-compose.yml"
             dest: "{{ grist_dir }}/docker-compose.yml"
             mode: "0644"
+            force: true
 
         # Pulled from Infisical, not generated/persisted to disk. Run this
         # playbook as `infisical run -- ansible-playbook -i inventory ...`

ansible/plays/machine_setup/nametag.yml

@@ -24,21 +24,28 @@
 
     - name: Deploy Nametag
       vars:
+        # Keep this path: compose prefixes named volumes (postgres_data etc.)
+        # with the directory name, so moving it orphans the existing data.
         nametag_dir: /opt/nametag
+        # Branch of AtlasHorizon/compose to deploy from; override with
+        # `-e compose_branch=<branch>` to test unmerged changes.
+        compose_branch: main
       block:
-        - name: Clone nametag repo
-          ansible.builtin.git:
-            repo: https://github.com/mattogodoy/nametag.git
-            dest: "{{ nametag_dir }}"
-            version: master
-            update: yes
-
         - name: Create backups directory
           ansible.builtin.file:
             path: "{{ nametag_dir }}/backups"
             state: directory
             mode: "0755"
 
+        # force: true so changes in the compose repo are picked up on reruns;
+        # it only reports changed when the content differs.
+        - name: Download docker-compose.yml
+          ansible.builtin.get_url:
+            url: "https://git.atlashorizon.net/AtlasHorizon/compose/raw/branch/{{ compose_branch }}/nametag/docker-compose.yml"
+            dest: "{{ nametag_dir }}/docker-compose.yml"
+            mode: "0644"
+            force: true
+
         # Pulled from Infisical, not generated/persisted to disk. Run this
         # playbook as `infisical run -- ansible-playbook -i inventory ...`
         # with NAMETAG_DB_PASSWORD, NAMETAG_NEXTAUTH_SECRET,

ansible/plays/machine_setup/supernote.yml

@@ -1,132 +0,0 @@
----
-- name: Set up Supernote Private Cloud
-  hosts: supernote
-  become: yes
-  tasks:
-    - name: default_packages
-      ansible.builtin.include_tasks: ../../templates/default_packages.yml
-
-    - name: create_user
-      ansible.builtin.include_tasks: ../../templates/create_user.yml
-      vars:
-        username: nick
-        github_user: Chewbaccalakis
-
-    - name: Install Docker
-      ansible.builtin.include_tasks: ../../templates/install_docker.yml
-
-    - name: Add nick to docker group
-      ansible.builtin.user:
-        name: nick
-        groups:
-          - docker
-        append: true
-
-    - name: Deploy Supernote Private Cloud
-      vars:
-        supernote_dir: /opt/supernote
-      block:
-        - name: Create data persistence directories
-          ansible.builtin.file:
-            path: "{{ supernote_dir }}/{{ item }}"
-            state: directory
-            mode: "0755"
-          loop:
-            - sndata/db_data
-            - sndata/config/mysql/conf.d
-            - sndata/redis_data
-            - sndata/recycle
-            - sndata/logs/cloud
-            - sndata/logs/app
-            - sndata/logs/web
-            - sndata/convert
-            - supernote_data
-
-        # Downloaded fresh each run so upstream schema changes apply on
-        # reruns, per Supernote's own upgrade instructions. docker-compose.yml
-        # is vendored instead (see templates/supernote-docker-compose.yml) -
-        # the upstream file is missing the initdb mounts this playbook needs.
-        - name: Download Supernote database init script
-          ansible.builtin.get_url:
-            url: https://supernote-private-cloud.supernote.com/cloud/supernotedb.sql
-            dest: "{{ supernote_dir }}/supernotedb.sql"
-            mode: "0644"
-            force: yes
-
-        - name: Deploy docker-compose.yml
-          ansible.builtin.copy:
-            src: templates/supernote-docker-compose.yml
-            dest: "{{ supernote_dir }}/docker-compose.yml"
-            mode: "0644"
-
-        # Pulled from Infisical, not generated/persisted to disk. Run this
-        # playbook as `infisical run -- ansible-playbook -i inventory ...`
-        # with SUPERNOTE_MYSQL_ROOT_PASSWORD, SUPERNOTE_MYSQL_PASSWORD,
-        # SUPERNOTE_REDIS_PASSWORD, SUPERNOTE_SMTP_SERVER, SUPERNOTE_SMTP_USER,
-        # and SUPERNOTE_SMTP_PASSWORD secrets set in the workspace.
-        # SUPERNOTE_SMTP_PORT (default 465) and SUPERNOTE_SMTP_ENCRYPTION
-        # (default ssl) are optional.
-        - name: Read secrets from environment (populated by `infisical run`)
-          ansible.builtin.set_fact:
-            supernote_mysql_root_password: "{{ lookup('ansible.builtin.env', 'SUPERNOTE_MYSQL_ROOT_PASSWORD') }}"
-            supernote_mysql_password: "{{ lookup('ansible.builtin.env', 'SUPERNOTE_MYSQL_PASSWORD') }}"
-            supernote_redis_password: "{{ lookup('ansible.builtin.env', 'SUPERNOTE_REDIS_PASSWORD') }}"
-            supernote_smtp_server: "{{ lookup('ansible.builtin.env', 'SUPERNOTE_SMTP_SERVER') }}"
-            supernote_smtp_port: "{{ lookup('ansible.builtin.env', 'SUPERNOTE_SMTP_PORT') }}"
-            supernote_smtp_user: "{{ lookup('ansible.builtin.env', 'SUPERNOTE_SMTP_USER') }}"
-            supernote_smtp_password: "{{ lookup('ansible.builtin.env', 'SUPERNOTE_SMTP_PASSWORD') }}"
-            supernote_smtp_encryption: "{{ lookup('ansible.builtin.env', 'SUPERNOTE_SMTP_ENCRYPTION') }}"
-          no_log: true
-
-        - name: Verify required secrets were provided
-          ansible.builtin.assert:
-            that:
-              - supernote_mysql_root_password | length > 0
-              - supernote_mysql_password | length > 0
-              - supernote_redis_password | length > 0
-              - supernote_smtp_server | length > 0
-              - supernote_smtp_user | length > 0
-              - supernote_smtp_password | length > 0
-            fail_msg: >-
-              SUPERNOTE_MYSQL_ROOT_PASSWORD, SUPERNOTE_MYSQL_PASSWORD,
-              SUPERNOTE_REDIS_PASSWORD, SUPERNOTE_SMTP_SERVER,
-              SUPERNOTE_SMTP_USER, and SUPERNOTE_SMTP_PASSWORD must be set.
-              Run this playbook via `infisical run -- ansible-playbook -i
-              inventory ...` with those secrets defined in the workspace.
-
-        - name: Deploy .env file (docker compose variable substitution)
-          ansible.builtin.template:
-            src: templates/supernote.env.j2
-            dest: "{{ supernote_dir }}/.env"
-            mode: "0600"
-
-        - name: Deploy SMTP config init script
-          ansible.builtin.template:
-            src: templates/supernote-email-init.sql.j2
-            dest: "{{ supernote_dir }}/supernote-email-init.sql"
-            mode: "0600"
-
-        - name: Pull prebuilt Supernote images
-          ansible.builtin.command:
-            cmd: docker compose pull
-            chdir: "{{ supernote_dir }}"
-          changed_when: false
-
-        - name: Start Supernote Private Cloud via docker compose
-          ansible.builtin.command:
-            cmd: docker compose up -d
-            chdir: "{{ supernote_dir }}"
-          register: supernote_compose_up
-          changed_when: "'Started' in supernote_compose_up.stderr or 'Created' in supernote_compose_up.stderr"
-
-        # docker-entrypoint-initdb.d scripts only run once, against an empty
-        # database, so reruns wouldn't otherwise pick up SMTP changes made in
-        # Infisical. Re-applying here keeps Infisical the source of truth.
-        - name: Apply SMTP configuration to the database
-          ansible.builtin.shell:
-            cmd: >-
-              docker compose exec -T -e MYSQL_PWD="{{ supernote_mysql_root_password }}"
-              mariadb mysql -u root supernotedb < {{ supernote_dir }}/supernote-email-init.sql
-            chdir: "{{ supernote_dir }}"
-          changed_when: false
-          no_log: true

ansible/plays/machine_setup/templates/grist-docker-compose.yml

@@ -1,12 +0,0 @@
-# Managed by Ansible (grist.yml). Vendored here since grist-core has no
-# docker-compose.yml at its repo root (see docker-compose-examples/ upstream).
-services:
-  grist:
-    image: gristlabs/grist:latest
-    restart: unless-stopped
-    ports:
-      - "8484:8484"
-    volumes:
-      - ./persist:/persist
-    env_file:
-      - .env

ansible/plays/machine_setup/templates/omnivore.env.j2

@@ -1,72 +0,0 @@
-# Managed by Ansible (omnivore.yml) - generated secrets persist in
-# ansible/plays/machine_setup/credentials/omnivore/ on the control node.
-
-# Postgres & Migrate
-
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD={{ omnivore_postgres_password }}
-POSTGRES_DB=omnivore
-PGPASSWORD={{ omnivore_postgres_password }}
-PG_REPLICATION=false
-
-PG_HOST=postgres
-PG_PASSWORD={{ omnivore_pg_password }}
-PG_DB=omnivore
-PG_USER=app_user
-PG_PORT=5432
-PG_POOL_MAX=20
-
-# API
-
-API_ENV=local
-IMAGE_PROXY_SECRET={{ omnivore_image_proxy_secret }}
-JWT_SECRET={{ omnivore_jwt_secret }}
-SSO_JWT_SECRET={{ omnivore_sso_jwt_secret }}
-GATEWAY_URL=http://api:8080/api
-CONTENT_FETCH_URL=http://content-fetch:8080/?token={{ omnivore_verification_token }}
-GCS_USE_LOCAL_HOST=true
-GCS_UPLOAD_BUCKET=omnivore
-AUTO_VERIFY=true
-# Minio credentials are hardcoded in docker-compose.yml for the minio and
-# createbuckets services, so these must match those literal values.
-AWS_ACCESS_KEY_ID=minio
-AWS_SECRET_ACCESS_KEY=miniominio
-AWS_REGION=us-east-1
-CONTENT_FETCH_QUEUE_ENABLED=true
-
-IMAGE_PROXY_URL=http://{{ ansible_host }}:7070
-CLIENT_URL=http://{{ ansible_host }}:3000
-LOCAL_MINIO_URL=http://{{ ansible_host }}:1010
-
-# Redis
-REDIS_URL=redis://redis:6379/0
-
-# Mail
-WATCHER_API_KEY={{ omnivore_mail_watcher_key }}
-LOCAL_EMAIL_DOMAIN=domain.tld
-# SNS_ARN=arn_of_sns #for if you use SES and SNS for Email.
-
-# Web
-APP_ENV=prod
-BASE_URL=http://{{ ansible_host }}:3000
-SERVER_BASE_URL=http://{{ ansible_host }}:4000
-HIGHLIGHTS_BASE_URL=http://{{ ansible_host }}:3000
-# USE_DISCOVER_AI=true
-# OPENAI_API_KEY=<OPENAI_API_KEY>
-
-# Content Fetch
-VERIFICATION_TOKEN={{ omnivore_verification_token }}
-REST_BACKEND_ENDPOINT=http://api:8080/api
-SKIP_UPLOAD_ORIGINAL=true
-
-# Minio
-MINIO_ACCESS_KEY=minio
-MINIO_SECRET_KEY=miniominio
-AWS_S3_ENDPOINT_URL=http://minio:9000
-
-# Export
-EXPORT_QUEUE_NAME=omnivore-backend-queue
-
-# Youtube
-YOUTUBE_TRANSCRIPT_PROMPT=Reformat the supplied transcript data adding formatting, puncuation, and paragraphs, but do not summarize or change the content. Format the output as markdown.
-YOUTUBE_MAXIMUM_VIDEO_DURATION_TRANSCRIPT=1801

ansible/plays/machine_setup/templates/supernote-docker-compose.yml

@@ -1,101 +0,0 @@
-# Vendored from https://supernote-private-cloud.supernote.com/cloud/docker-compose.yml,
-# with two fixes applied:
-#   - the mariadb service is missing docker-entrypoint-initdb.d mounts in the
-#     upstream file, so supernotedb.sql would never actually load into a
-#     fresh database; added below.
-#   - a second init script (supernote-email-init.sql, rendered by Ansible
-#     from Infisical-sourced secrets) pre-seeds SMTP settings into
-#     u_email_config so registration/password-reset emails work without a
-#     manual trip through the web UI's "Email Settings" dialog.
-# supernote-service's DB_HOSTNAME/MYSQL_*/REDIS_* env vars are also inlined
-# directly here instead of upstream's separate .dbenv/env_file, since they're
-# just ${VAR} substitutions from the same .env used by mariadb/redis.
-services:
-  mariadb:
-    image: mariadb:10.6.24
-    container_name: mariadb
-    environment:
-      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
-      MYSQL_DATABASE: supernotedb
-      MYSQL_USER: ${MYSQL_USER}
-      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
-      TZ: ${TZ:-UTC}
-    command: --skip-name-resolve --bind-address=0.0.0.0
-    volumes:
-      - ./sndata/db_data:/var/lib/mysql
-      - ./sndata/config/mysql/conf.d:/etc/mysql/conf.d:ro
-      - ./supernotedb.sql:/docker-entrypoint-initdb.d/00-supernotedb.sql:ro
-      - ./supernote-email-init.sql:/docker-entrypoint-initdb.d/01-supernote-email-init.sql:ro
-      - /etc/localtime:/etc/localtime:ro
-    healthcheck:
-      test: ["CMD-SHELL", "mysqladmin ping -h localhost -u root -p$$MYSQL_ROOT_PASSWORD || exit 1"]
-      interval: 10s
-      timeout: 5s
-      retries: 10
-      start_period: 30s
-    restart: unless-stopped
-
-  redis:
-    image: redis:7.4.7
-    container_name: redis
-    environment:
-      TZ: ${TZ:-UTC}
-    command: ["redis-server", "--requirepass", "${REDIS_PASSWORD}"]
-    volumes:
-      - ./sndata/redis_data:/data
-      - /etc/localtime:/etc/localtime:ro
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 10s
-      timeout: 3s
-      retries: 3
-
-  notelib:
-    image: docker.io/supernote/notelib:6.9.3
-    container_name: notelib
-    environment:
-      TZ: ${TZ:-UTC}
-    volumes:
-      - /etc/localtime:/etc/localtime:ro
-    restart: unless-stopped
-
-  supernote-service:
-    image: docker.io/supernote/supernote-service:26.06.04
-    container_name: supernote-service
-    depends_on:
-      mariadb:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    ports:
-      - "19071:19071"
-      - "18072:18072"
-      - "${FRONTEND_PORT:-19072}:8080"
-      - "${DOMAIN_PORT:-8199}:8080"
-    environment:
-      EXTERNAL_PORT_BACKEND: "19071"
-      EXTERNAL_PORT_FRONTEND: ${FRONTEND_PORT:-19072}
-      EXTERNAL_PORT_DOMAIN: ${DOMAIN_PORT:-8199}
-      TZ: ${TZ:-UTC}
-      DB_HOSTNAME: mariadb
-      MYSQL_DATABASE: supernotedb
-      MYSQL_USER: ${MYSQL_USER}
-      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      REDIS_PASSWORD: ${REDIS_PASSWORD}
-    volumes:
-      - ./sndata/recycle:/home/supernote/recycle
-      - ./supernote_data:/home/supernote/data
-      - ./sndata/logs/cloud:/home/supernote/cloud/logs
-      - ./sndata/logs/app:/home/supernote/logs
-      - ./sndata/logs/web:/var/log/nginx
-      - ./sndata/convert:/home/supernote/convert
-      - /etc/localtime:/etc/localtime:ro
-    restart: unless-stopped
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "10m"
-        max-file: "3"

ansible/plays/machine_setup/templates/supernote-email-init.sql.j2

@@ -1,30 +0,0 @@
--- Managed by Ansible (supernote.yml) - SMTP secrets sourced from Infisical
--- via `infisical run -- ansible-playbook ...`.
---
--- Mounted into the mariadb container's docker-entrypoint-initdb.d/ so it
--- seeds u_email_config on first boot, and re-applied by the playbook via
--- `docker compose exec` on every run so Infisical stays the source of truth.
---
--- u_email_config's schema/upsert key come from a third-party reverse
--- engineering of Supernote's supernotedb.sql (not officially documented) -
--- see https://github.com/liketheduck/supernote-private-cloud-docker. Verify
--- against Email Settings in the web UI if Supernote changes this schema.
-
-USE supernotedb;
-
-INSERT INTO u_email_config (smtp_server, port, user_name, password, encryption, flag)
-VALUES (
-  '{{ supernote_smtp_server }}',
-  '{{ supernote_smtp_port | default(465) }}',
-  '{{ supernote_smtp_user }}',
-  '{{ supernote_smtp_password }}',
-  '{{ supernote_smtp_encryption | default("ssl") }}',
-  'Y'
-)
-ON DUPLICATE KEY UPDATE
-  smtp_server='{{ supernote_smtp_server }}',
-  port='{{ supernote_smtp_port | default(465) }}',
-  user_name='{{ supernote_smtp_user }}',
-  password='{{ supernote_smtp_password }}',
-  encryption='{{ supernote_smtp_encryption | default("ssl") }}',
-  flag='Y';

ansible/plays/machine_setup/templates/supernote.env.j2

@@ -1,10 +0,0 @@
-# Managed by Ansible (supernote.yml) - secrets sourced from Infisical via
-# `infisical run -- ansible-playbook ...`.
-#
-# Consumed by docker compose for ${VAR} substitution in docker-compose.yml.
-
-MYSQL_ROOT_PASSWORD={{ supernote_mysql_root_password }}
-MYSQL_USER=supernote
-MYSQL_PASSWORD={{ supernote_mysql_password }}
-
-REDIS_PASSWORD={{ supernote_redis_password }}

backend.tf

@@ -8,9 +8,9 @@ terraform {
   backend "s3" {
     bucket = "terraform"
     key    = "infra/terraform.tfstate"
-    region = "garage"
+    region = "us-east-1"
 
-    endpoint         = "http://s3.lan.oswaldtheotter.net:30188"
+    endpoint         = "https://goldie.lan.atlashorizon.net:30292"
     use_path_style = true
 
     skip_credentials_validation = true

butane/ai.yaml.tftpl

@@ -8,9 +8,12 @@ systemd:
       contents: |
         [Unit]
         Description=Gitea MCP Server
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force gitea-mcp
         # The image ships a broken HEALTHCHECK: it is CMD-SHELL form but the
@@ -21,7 +24,7 @@ systemd:
         ExecStart=/usr/bin/docker run --name gitea-mcp --no-healthcheck \
           -e GITEA_HOST=https://git.atlashorizon.net \
           -p 8080:8080 \
-          docker.gitea.com/gitea-mcp-server \
+          $${GITEA_MCP_IMAGE} \
           /app/gitea-mcp -t http --port 8080
         ExecStop=/usr/bin/docker stop gitea-mcp
         Restart=always
@@ -33,9 +36,12 @@ systemd:
       contents: |
         [Unit]
         Description=LiteLLM proxy
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force litellm
         ExecStart=/usr/bin/docker run --name litellm \
@@ -45,112 +51,380 @@ systemd:
           -e LITELLM_SALT_KEY=${litellm_salt_key} \
           -e LITELLM_LOG=ERROR \
           -e JSON_LOGS=True \
+          -v /etc/litellm/config.yaml:/app/config.yaml:ro \
           -p 4000:4000 \
-          docker.litellm.ai/berriai/litellm:main-stable
+          $${LITELLM_IMAGE} \
+          --config /app/config.yaml --port 4000
         ExecStop=/usr/bin/docker stop litellm
         Restart=always
         RestartSec=5s
         [Install]
         WantedBy=multi-user.target
-    # ── Open WebUI (disabled) ──────────────────────────────────────────────
-    # Commented out until it is ready to launch. To re-enable:
-    #   1. strip the leading "# " from every line of the block below (this
-    #      restores the original indentation exactly)
-    #   2. un-double the dollar signs in each interpolation: $$ back to $
-    #   3. re-add the seven OPENWEBUI_* keys to secrets.tmpl (listed below;
-    #      they are removed while this is off so the generated JSON, which
-    #      data.external parses, stays valid)
+    # Open WebUI, with Qdrant alongside it for document embeddings.
     #
-    #      OPENWEBUI_DATABASE_URL        -> openwebui_database_url
-    #      OPENWEBUI_SECRET_KEY          -> openwebui_secret_key
-    #      OPENWEBUI_S3_ACCESS_KEY       -> openwebui_s3_access_key
-    #      OPENWEBUI_S3_SECRET_KEY       -> openwebui_s3_secret_key
-    #      OPENWEBUI_OAUTH_CLIENT_ID     -> openwebui_oauth_client_id
-    #      OPENWEBUI_OAUTH_CLIENT_SECRET -> openwebui_oauth_client_secret
-    #      OPENWEBUI_OIDC_PROVIDER_URL   -> openwebui_oidc_provider_url
+    # Prerequisites, all of which must exist before the VM is rebuilt:
+    #   - Postgres database (plain; no pgvector extension needed -- embeddings
+    #     go to Qdrant, and Postgres only holds users, chats and settings)
+    #   - openwebui bucket in Garage, plus a key with r/w on it
+    #   - an embedding model in LiteLLM matching RAG_EMBEDDING_MODEL below
+    #   - an OIDC client at the provider, with the callback URI registered as
+    #     https://llm.atlashorizon.net/oauth/oidc/callback
+    #   - llm.atlashorizon.net terminating TLS and proxying to this host's
+    #     :3000 (not managed here -- see WEBUI_URL below)
+    #   - the OPENWEBUI_* secrets in Infisical (see secrets.tmpl)
     #
-    # The doubling in step 2 is not cosmetic: templatefile() substitutes on
-    # raw text before anything parses it as YAML, so an interpolation left
-    # intact here would still be resolved -- and would still fail the plan on
-    # a missing key -- even though it sits inside a comment.
-#     # Open WebUI.
-#     #
-#     # Prerequisites, all of which must exist before the VM is rebuilt:
-#     #   - Postgres database, with "CREATE EXTENSION vector;" run inside it
-#     #   - openwebui bucket in Garage, plus a key with r/w on it
-#     #   - an embedding model in LiteLLM matching RAG_EMBEDDING_MODEL below
-#     #   - an OIDC client at the provider, with the callback URI registered as
-#     #     https://llm.atlashorizon.net/oauth/oidc/callback
-#     #   - llm.atlashorizon.net terminating TLS and proxying to this host's
-#     #     :3000 (not managed here -- see WEBUI_URL below)
-#     #   - the OPENWEBUI_* secrets in Infisical (see secrets.tmpl)
-#     - name: open-webui.service
-#       enabled: true
-#       contents: |
-#         [Unit]
-#         Description=Open WebUI
-#         After=docker.service litellm.service
-#         Requires=docker.service
-#         [Service]
-#         TimeoutStartSec=0
-#         # Large image, so it is pulled up front: a slow first pull is then not
-#         # mistaken for a hung unit.
-#         ExecStartPre=/usr/bin/docker pull ghcr.io/open-webui/open-webui:v0.11.4
-#         ExecStartPre=-/usr/bin/docker rm --force open-webui
-#         # No volume is mounted. Every piece of durable state lives off-box --
-#         # Postgres for app data, pgvector for embeddings, Garage for uploads --
-#         # so /app/backend/data holds nothing worth keeping and the container
-#         # (and the whole VM) can be rebuilt at will. What stays local is cache
-#         # only, re-fetched on the next start.
-#         #
-#         # gitea-mcp already owns host :8080, so the UI is published on :3000.
-#         ExecStart=/usr/bin/docker run --name open-webui \
-#           -e DATABASE_URL=$${openwebui_database_url} \
-#           -e VECTOR_DB=pgvector \
-#           -e PGVECTOR_DB_URL=$${openwebui_database_url} \
-#           -e STORAGE_PROVIDER=s3 \
-#           -e S3_ENDPOINT_URL=http://s3.lan.oswaldtheotter.net:30188 \
-#           -e S3_REGION_NAME=garage \
-#           -e S3_BUCKET_NAME=openwebui \
-#           -e S3_ACCESS_KEY_ID=$${openwebui_s3_access_key} \
-#           -e S3_SECRET_ACCESS_KEY=$${openwebui_s3_secret_key} \
-#           -e WEBUI_SECRET_KEY=$${openwebui_secret_key} \
-#           -e ENABLE_OLLAMA_API=false \
-#           -e OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \
-#           -e OPENAI_API_KEY=$${litellm_master_key} \
-#           -e RAG_EMBEDDING_ENGINE=openai \
-#           -e RAG_OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \
-#           -e RAG_OPENAI_API_KEY=$${litellm_master_key} \
-#           -e RAG_EMBEDDING_MODEL=text-embedding-3-small \
-#           -e OAUTH_CLIENT_ID=$${openwebui_oauth_client_id} \
-#           -e OAUTH_CLIENT_SECRET=$${openwebui_oauth_client_secret} \
-#           -e OPENID_PROVIDER_URL=$${openwebui_oidc_provider_url} \
-#           -e OAUTH_PROVIDER_NAME=SSO \
-#           -e ENABLE_OAUTH_SIGNUP=true \
-#           -e WEBUI_URL=https://llm.atlashorizon.net \
-#           -p 3000:8080 \
-#           ghcr.io/open-webui/open-webui:v0.11.4
-#         ExecStop=/usr/bin/docker stop open-webui
-#         Restart=always
-#         RestartSec=5s
-#         [Install]
-#         WantedBy=multi-user.target
+    # After a taint: the vector store is empty and Open WebUI does not say so
+    # -- knowledge bases still appear, retrieval just returns nothing. Re-embed
+    # with POST /api/v1/knowledge/reindex (admin). Note that this covers
+    # knowledge bases only: files attached directly to a single chat are not
+    # re-indexed, so anything worth keeping belongs in a knowledge base.
+    - name: qdrant.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Qdrant vector database
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
+        Requires=docker.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        # Both containers sit on a private docker network and Qdrant publishes
+        # no ports: it ships with no authentication, and nothing outside this
+        # host needs to reach it. Creation is idempotent, hence the "-".
+        ExecStartPre=-/usr/bin/docker network create openwebui
+        ExecStartPre=/usr/bin/docker pull $${QDRANT_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force qdrant
+        # Vectors live on the VM disk, so a taint destroys them. That is
+        # deliberate -- they are derived data, rebuildable from the source
+        # files in Garage (see the reindex note above). The bind mount is what
+        # keeps them across container restarts and image pulls, which the
+        # docker rm above would otherwise wipe every time.
+        ExecStart=/usr/bin/docker run --name qdrant \
+          --network openwebui \
+          -v /var/lib/qdrant:/qdrant/storage \
+          $${QDRANT_IMAGE}
+        ExecStop=/usr/bin/docker stop qdrant
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # Open Terminal: the sandbox Open WebUI's models get a shell in, for
+    # running code, installing packages and working on files.
+    #
+    # Anything a model does lands inside this container, not on the host: no
+    # docker socket, no host paths, no published port. It is reachable only
+    # over the private "openwebui" network, so the API key below is defence in
+    # depth rather than the only thing standing in the way.
+    #
+    # Note that it does have outbound network access -- it needs it to install
+    # packages -- so a model can reach the rest of the LAN from in here. That
+    # is the part worth thinking about before pointing an untrusted model at
+    # it; docker's --internal would close it off, but would also break package
+    # installation, which is most of the point.
+    - name: open-terminal.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Open Terminal sandbox
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
+        Requires=docker.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        ExecStartPre=-/usr/bin/docker network create openwebui
+        ExecStartPre=/usr/bin/docker pull $${OPEN_TERMINAL_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force open-terminal
+        # A named volume, not a bind mount: /home/user belongs to the image's
+        # unprivileged user, and a named volume inherits that ownership from
+        # the image where a root-owned bind mount would not. It lives under
+        # /var/lib/docker on the VM disk, so a taint discards the workspace --
+        # the right lifetime for scratch space.
+        ExecStart=/usr/bin/docker run --name open-terminal \
+          --network openwebui \
+          -v open-terminal:/home/user \
+          -e OPEN_TERMINAL_API_KEY=${openwebui_terminal_api_key} \
+          $${OPEN_TERMINAL_IMAGE}
+        ExecStop=/usr/bin/docker stop open-terminal
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    - name: open-webui.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Open WebUI
+        # After= orders the start only, it does not wait for Qdrant to be
+        # serving. Restart=always covers the gap if Qdrant is still coming up.
+        After=docker.service litellm.service qdrant.service open-terminal.service versions-fetch.service
+        Wants=versions-fetch.service
+        Requires=docker.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        # Large image, so it is pulled up front: a slow first pull is then not
+        # mistaken for a hung unit.
+        ExecStartPre=/usr/bin/docker pull $${OPEN_WEBUI_IMAGE}
+        ExecStartPre=-/usr/bin/docker network create openwebui
+        ExecStartPre=-/usr/bin/docker rm --force open-webui
+        # No volume is mounted. Every piece of durable state lives off-box --
+        # Postgres for app data, Garage for uploads -- so /app/backend/data
+        # holds nothing worth keeping and the container can be recreated at
+        # will. Qdrant is the one exception, and it is expendable by design.
+        #
+        # gitea-mcp already owns host :8080, so the UI is published on :3000.
+        ExecStart=/usr/bin/docker run --name open-webui \
+          --network openwebui \
+          -e DATABASE_URL=${openwebui_database_url} \
+          -e VECTOR_DB=qdrant \
+          -e QDRANT_URI=http://qdrant:6333 \
+          -e QDRANT_ON_DISK=true \
+          -e STORAGE_PROVIDER=s3 \
+          -e S3_ENDPOINT_URL=http://s3.lan.oswaldtheotter.net:30188 \
+          -e S3_REGION_NAME=garage \
+          -e S3_BUCKET_NAME=openwebui \
+          -e S3_ACCESS_KEY_ID=${openwebui_s3_access_key} \
+          -e S3_SECRET_ACCESS_KEY=${openwebui_s3_secret_key} \
+          -e WEBUI_SECRET_KEY=${openwebui_secret_key} \
+          -e ENABLE_OLLAMA_API=false \
+          -e OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \
+          -e OPENAI_API_KEY=${litellm_master_key} \
+          -e RAG_EMBEDDING_ENGINE=openai \
+          -e RAG_OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \
+          -e RAG_OPENAI_API_KEY=${litellm_master_key} \
+          -e RAG_EMBEDDING_MODEL=text-embedding-3-small \
+          -e OAUTH_CLIENT_ID=${openwebui_oauth_client_id} \
+          -e OAUTH_CLIENT_SECRET=${openwebui_oauth_client_secret} \
+          -e OPENID_PROVIDER_URL=${openwebui_oidc_provider_url} \
+          -e OAUTH_PROVIDER_NAME=AtlasHorizon \
+          -e ENABLE_OAUTH_SIGNUP=true \
+          -e OAUTH_UPDATE_PICTURE_ON_LOGIN=true \
+          -e ENABLE_OAUTH_GROUP_MANAGEMENT=true \
+          -e OAUTH_PICTURE_CLAIM=avatar \
+          -e "OAUTH_SCOPES=openid email profile groups avatar" \
+          -e OAUTH_MERGE_ACCOUNTS_BY_EMAIL=true \
+          -e CORS_ALLOW_ORIGIN=https://llm.atlashorizon.net \
+          -e WEBUI_URL=https://llm.atlashorizon.net \
+          -e 'TERMINAL_SERVER_CONNECTIONS=[{"id":"local","name":"Open Terminal","url":"http://open-terminal:8000","key":"${openwebui_terminal_api_key}","auth_type":"bearer","enabled":true}]' \
+          -p 3000:8080 \
+          $${OPEN_WEBUI_IMAGE}
+        ExecStop=/usr/bin/docker stop open-webui
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # OpenCode, run headless as a coding agent that n8n drives over HTTP.
+    #
+    # Prerequisites, all of which must exist before the VM is rebuilt:
+    #   - OPENCODE_SERVER_PASSWORD in Infisical (see secrets.tmpl). Every
+    #     Butane file is rendered on every plan, so until the secret exists
+    #     no plan succeeds -- not just this VM's
+    #   - each model listed in /etc/opencode/opencode.json (below) defined
+    #     in LiteLLM under the same model_name
+    #
+    # Unlike Open Terminal, this publishes a port: n8n runs on oswald-runner,
+    # so the private "openwebui" network is out of its reach. That leaves HTTP
+    # Basic Auth (user "opencode") as the only thing between the LAN and an
+    # agent with a shell, and over plain HTTP. The agent runs as root inside
+    # the container, with the same outbound network access as Open Terminal.
+    #
+    # From n8n: http://192.168.7.61:4096, with the n8n-nodes-opencode-ai
+    # community node or HTTP Request nodes (the OpenAPI spec is at /doc).
+    - name: opencode.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=OpenCode server
+        After=docker.service litellm.service versions-fetch.service
+        Wants=versions-fetch.service
+        Requires=docker.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        ExecStartPre=/usr/bin/docker pull $${OPENCODE_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force opencode
+        # Both directories are on the VM disk, so they survive container
+        # restarts but not a taint. workspace is what the agent edits; data
+        # holds its sessions, which n8n refers to by id across executions.
+        #
+        # The LiteLLM key is passed in the environment and read by the config
+        # file as {env:LITELLM_API_KEY}, which keeps the file free of secrets.
+        ExecStart=/usr/bin/docker run --name opencode \
+          -e OPENCODE_SERVER_USERNAME=opencode \
+          -e OPENCODE_SERVER_PASSWORD=${opencode_server_password} \
+          -e LITELLM_API_KEY=${litellm_master_key} \
+          -e OPENCODE_CONFIG=/etc/opencode/opencode.json \
+          -v /etc/opencode/opencode.json:/etc/opencode/opencode.json:ro \
+          -v /var/lib/opencode/workspace:/workspace \
+          -v /var/lib/opencode/data:/root/.local/share/opencode \
+          -w /workspace \
+          -p 4096:4096 \
+          $${OPENCODE_IMAGE} \
+          serve --hostname 0.0.0.0 --port 4096
+        ExecStop=/usr/bin/docker stop opencode
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # Rebuilds the vector store after a taint. Runs on every boot, but only
+    # acts when Qdrant holds no Open WebUI collections -- that is, when the
+    # local volume went with the VM.
+    #
+    # The admin API key it authenticates with lives in Postgres, so it
+    # survives the rebuild that wipes Qdrant. That is what makes this work.
+    - name: openwebui-reindex.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Rebuild Open WebUI vectors when Qdrant is empty
+        After=open-webui.service qdrant.service
+        Requires=open-webui.service qdrant.service
+        [Service]
+        Type=oneshot
+        TimeoutStartSec=30min
+        ExecStart=/opt/bin/openwebui-reindex.sh
+        [Install]
+        WantedBy=multi-user.target
     - name: dozzle-agent.service
       enabled: true
       contents: |
         [Unit]
         Description=Dozzle Agent
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force dozzle-agent
         ExecStart=/usr/bin/docker run --name dozzle-agent \
           -v /var/run/docker.sock:/var/run/docker.sock \
           -p 7007:7007 \
-          amir20/dozzle:v11.1.1 agent
+          $${DOZZLE_IMAGE} agent
         ExecStop=/usr/bin/docker stop dozzle-agent
         Restart=always
         RestartSec=5s
         [Install]
         WantedBy=multi-user.target
+
+storage:
+  directories:
+    # Qdrant's vector store. On the VM disk on purpose: a taint is meant to
+    # discard it. The qdrant image runs as root (ARG USER_ID=0), so a
+    # root-owned 0700 directory is what it expects.
+    - path: /var/lib/qdrant
+      mode: 0700
+    # OpenCode's workspace and session store; see opencode.service. The
+    # image runs as root.
+    - path: /var/lib/opencode/workspace
+      mode: 0755
+    - path: /var/lib/opencode/data
+      mode: 0700
+  files:
+    # Companion to openwebui-reindex.service above.
+    #
+    # Note on quoting: templatefile() substitutes on raw text, so shell
+    # variables are written as $VAR without braces. A braced shell variable
+    # anywhere in this script would be read as a template reference and fail
+    # the plan; only the interpolation below is meant to be substituted.
+    - path: /opt/bin/openwebui-reindex.sh
+      mode: 0700
+      contents:
+        inline: |
+          #!/usr/bin/bash
+          set -euo pipefail
+          OWUI=http://127.0.0.1:3000
+          OWUI_KEY='${openwebui_admin_api_key}'
+
+          # Qdrant publishes no ports -- it is reachable only on the private
+          # "openwebui" docker network -- so it is probed from inside the
+          # open-webui container, which is on that network and ships curl.
+          qdrant_curl() {
+            docker exec open-webui curl -fsS "$@"
+          }
+
+          # Wait up to 10 minutes for both services to answer.
+          ready=0
+          for i in $(seq 1 120); do
+            if curl -fsS "$OWUI/health" >/dev/null 2>&1 \
+               && qdrant_curl http://qdrant:6333/readyz >/dev/null 2>&1; then
+              ready=1; break
+            fi
+            sleep 5
+          done
+          [ "$ready" = 1 ] || { echo "services never became ready"; exit 1; }
+
+          # Collections are named "<QDRANT_COLLECTION_PREFIX>_<id>", and the
+          # prefix defaults to open-webui. Any hit means the volume survived.
+          if qdrant_curl http://qdrant:6333/collections \
+               | grep -q '"name":"open-webui'; then
+            echo "Qdrant already populated; skipping reindex"
+            exit 0
+          fi
+
+          echo "Qdrant empty; reindexing knowledge bases"
+          curl -fsS -X POST -H "Authorization: Bearer $OWUI_KEY" \
+            "$OWUI/api/v1/knowledge/reindex"
+    # LiteLLM proxy settings. Models stay in the database
+    # (STORE_MODEL_IN_DB), so this file carries general_settings only --
+    # those have no environment equivalent and are readable only from a
+    # config file.
+    #
+    # Requests reach :4000 through a Cloudflare tunnel whose connector runs
+    # on 192.168.0.0/24, so every request arrives from that connector's
+    # address. Without use_x_forwarded_for LiteLLM does MCP access control
+    # against that peer, which sits inside the default internal ranges, and
+    # so treats every caller on the public internet as internal -- exposing
+    # MCP servers marked "available_on_public_internet: false".
+    #
+    # mcp_trusted_proxy_ranges is what keeps that safe: X-Forwarded-For is
+    # honoured only when the direct peer is the connector, so a crafted
+    # header from anywhere else cannot forge a client IP. Widen it if the
+    # tunnel ever moves off that subnet, or the header is ignored again.
+    - path: /etc/litellm/config.yaml
+      mode: 0644
+      contents:
+        inline: |
+          general_settings:
+            use_x_forwarded_for: true
+            mcp_trusted_proxy_ranges:
+              - 192.168.0.0/24
+    # OpenCode's provider config. LiteLLM speaks the OpenAI API, so it is
+    # wired in through @ai-sdk/openai-compatible.
+    #
+    # Model keys must match LiteLLM model_name values exactly. Those live in
+    # the LiteLLM database, not in a file here, so check GET :4000/v1/models
+    # when changing this list. "model" is the default for new sessions.
+    - path: /etc/opencode/opencode.json
+      mode: 0644
+      contents:
+        inline: |
+          {
+            "$schema": "https://opencode.ai/config.json",
+            "model": "litellm/claude-sonnet-5",
+            "provider": {
+              "litellm": {
+                "npm": "@ai-sdk/openai-compatible",
+                "name": "LiteLLM",
+                "options": {
+                  "baseURL": "http://192.168.7.61:4000/v1",
+                  "apiKey": "{env:LITELLM_API_KEY}"
+                },
+                "models": {
+                  "claude-sonnet-5": {
+                    "name": "Claude Sonnet 5"
+                  },
+                  "qwen3:4b-instruct": {
+                    "name": "Qwen3 4B Instruct"
+                  }
+                }
+              }
+            }
+          }

butane/cisa_kev_rss.yaml

@@ -1,52 +0,0 @@
-variant: flatcar
-version: 1.0.0
-
-# CISA KEV RSS feed. Turns the CISA Known Exploited Vulnerabilities catalog
-# into an RSS 2.0 feed (https://github.com/xsaveopt/cisa-kev-rss).
-#
-# Feed:  http://192.168.7.64:3000/rss
-# Health: http://192.168.7.64:3000/health
-#
-# Stateless: the container refreshes from CISA every ten minutes and keeps
-# nothing on disk, so it is safe to rebuild at any time.
-
-systemd:
-  units:
-    - name: cisa-kev-rss.service
-      enabled: true
-      contents: |
-        [Unit]
-        Description=CISA KEV RSS feed
-        After=docker.service
-        Requires=docker.service
-        [Service]
-        TimeoutStartSec=0
-        ExecStartPre=/usr/bin/docker pull ghcr.io/xsaveopt/cisa-kev-rss:latest
-        ExecStartPre=-/usr/bin/docker rm --force cisa-kev-rss
-        ExecStart=/usr/bin/docker run --rm --name cisa-kev-rss \
-          -p 3000:3000 \
-          ghcr.io/xsaveopt/cisa-kev-rss:latest
-        ExecStop=/usr/bin/docker stop cisa-kev-rss
-        Restart=always
-        RestartSec=5s
-        [Install]
-        WantedBy=multi-user.target
-    - name: dozzle-agent.service
-      enabled: true
-      contents: |
-        [Unit]
-        Description=Dozzle Agent
-        After=docker.service
-        Requires=docker.service
-        [Service]
-        TimeoutStartSec=0
-        ExecStartPre=-/usr/bin/docker rm --force dozzle-agent
-        ExecStart=/usr/bin/docker run --name dozzle-agent \
-          -v /var/run/docker.sock:/var/run/docker.sock \
-          -p 7007:7007 \
-          amir20/dozzle:v11.1.1 agent
-        ExecStop=/usr/bin/docker stop dozzle-agent
-        Restart=always
-        RestartSec=5s
-        [Install]
-        WantedBy=multi-user.target

butane/monitor.yaml.tftpl

@@ -13,6 +13,8 @@ version: 1.0.0
 # VMs (see the dozzle-agent units in the other butane configs) and, via the
 # local socket, from this host's own containers. Reachable on :8080.
 #
+# Also runs Alerta on :8082; see alerta.service below.
+#
 # No state is kept on disk: /data is a tmpfs, so the container is fully
 # disposable and there is nothing to back up. That is also why simple auth
 # (users.yml) is not used -- it would need a persistent /data.
@@ -20,10 +22,11 @@ version: 1.0.0
 # Hub <-> agent traffic is TLS, using the certificate pair baked into the
 # dozzle image. Every hub and agent on the same image version shares it, so
 # nothing needs mounting here -- which is also why the agents must stay pinned
-# to the same tag as the hub. To move to a custom pair later, generate one
-# with (writes into $PWD):
+# to the same tag as the hub (DOZZLE_IMAGE in versions.env). To move to a
+# custom pair later, generate one with (writes into $PWD, using the tag from
+# DOZZLE_IMAGE):
 #
-#   docker run --rm -v "$PWD":/out amir20/dozzle:v11.1.1 generate-certs \
+#   docker run --rm -v "$PWD":/out amir20/dozzle:<tag> generate-certs \
 #     --cert-out /out/dozzle_cert.pem --key-out /out/dozzle_key.pem
 #
 # ...then mount it at /dozzle_cert.pem and /dozzle_key.pem (read-only) on the
@@ -37,16 +40,20 @@ systemd:
       contents: |
         [Unit]
         Description=Dozzle log viewer hub
-        After=docker.service
+        After=docker.service versions-fetch.service
         Requires=docker.service
+        Wants=versions-fetch.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         # Kept as unit environment rather than inline -e values so that
         # wait-for-agents.sh can read the agent list from the same place the
         # container does. docker run forwards them by bare name below.
         Environment="DOZZLE_HOSTNAME=Oswald Logs"
-        Environment="DOZZLE_ENABLE_MCP=true"
-        Environment="DOZZLE_REMOTE_AGENT=192.168.7.60:7007|oswald-runner|Goose,192.168.7.61:7007|ai|Goose,192.168.7.62:7007|oswl-connector|Goose,192.168.7.64:7007|cisa-kev-rss|Goose"
+        # The agent list lives in a file because dozzle-mcp.service needs the
+        # identical list; keeping it inline in both units would drift.
+        EnvironmentFile=/etc/dozzle/agents.env
         # OIDC via Authelia. Dozzle builds its redirect from the incoming
         # request, so the proxy in front must pass the real Host and
         # X-Forwarded-Proto; the URI to register in Authelia is
@@ -62,7 +69,7 @@ systemd:
         Environment="DOZZLE_AUTH_OIDC_SCOPES=dozzle_roles,picture"
         Environment="DOZZLE_AUTH_OIDC_ROLES_CLAIM=dozzle_roles"
         # Pulled up front so a slow first pull is not mistaken for a hung unit.
-        ExecStartPre=/usr/bin/docker pull amir20/dozzle:v11.1.1
+        ExecStartPre=/usr/bin/docker pull $${DOZZLE_IMAGE}
         # Leading "-" so a missing container is not a failure.
         ExecStartPre=-/usr/bin/docker rm --force dozzle
         # If the hub starts before the agents are listening, the failed initial
@@ -76,7 +83,6 @@ systemd:
           --tmpfs /data \
           -p 8080:8080 \
           -e DOZZLE_HOSTNAME \
-          -e DOZZLE_ENABLE_MCP \
           -e DOZZLE_REMOTE_AGENT \
           -e DOZZLE_AUTH_PROVIDER \
           -e DOZZLE_AUTH_OIDC_ISSUER \
@@ -85,15 +91,128 @@ systemd:
           -e DOZZLE_AUTH_OIDC_NAME \
           -e DOZZLE_AUTH_OIDC_SCOPES \
           -e DOZZLE_AUTH_OIDC_ROLES_CLAIM \
-          amir20/dozzle:v11.1.1
+          $${DOZZLE_IMAGE}
         ExecStop=/usr/bin/docker stop dozzle
         Restart=always
         RestartSec=5s
         [Install]
         WantedBy=multi-user.target
+    # Second Dozzle, dedicated to MCP on :8081.
+    #
+    # It exists because the two requirements cannot be met by one instance:
+    # the oidc provider is the only one that lets Authelia own the user list,
+    # but it offers no non-interactive login, so an MCP client can never
+    # authenticate against it. This instance therefore runs with no auth at
+    # all (DOZZLE_AUTH_PROVIDER unset) and is reached only through the
+    # Cloudflare tunnel, which is where access control has to live.
+    #
+    # ANYTHING that can open :8081 gets unauthenticated read access to every
+    # log on every agent. Do not publish this port beyond the tunnel, and put
+    # a Cloudflare Access policy (service token) on its hostname.
+    - name: dozzle-mcp.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Dozzle MCP endpoint
+        After=docker.service versions-fetch.service
+        Requires=docker.service
+        Wants=versions-fetch.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        # Same agents as the hub, same file, so the two cannot drift.
+        EnvironmentFile=/etc/dozzle/agents.env
+        Environment="DOZZLE_HOSTNAME=Oswald Logs (MCP)"
+        Environment="DOZZLE_ENABLE_MCP=true"
+        ExecStartPre=/usr/bin/docker pull $${DOZZLE_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force dozzle-mcp
+        ExecStartPre=-/usr/bin/timeout 180 /opt/dozzle/wait-for-agents.sh
+        ExecStart=/usr/bin/docker run --rm --name dozzle-mcp \
+          -v /var/run/docker.sock:/var/run/docker.sock:ro \
+          --tmpfs /data \
+          -p 8081:8080 \
+          -e DOZZLE_HOSTNAME \
+          -e DOZZLE_REMOTE_AGENT \
+          -e DOZZLE_ENABLE_MCP \
+          $${DOZZLE_IMAGE}
+        ExecStop=/usr/bin/docker stop dozzle-mcp
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # Alerta alert console (API + web UI) on :8082.
+    #
+    # Prerequisites, all of which must exist before the VM is rebuilt:
+    #   - an "alerta" Postgres database and role; the schema is created on
+    #     first start
+    #   - an OIDC client for Alerta in Authelia, with the Alerta URL as a
+    #     redirect URI and token_endpoint_auth_method client_secret_basic
+    #   - ALERTA_DATABASE_URL, ALERTA_SECRET_KEY, ALERTA_ADMIN_PASSWORD,
+    #     ALERTA_OIDC_CLIENT_ID and ALERTA_OIDC_CLIENT_SECRET in
+    #     Infisical (see secrets.tmpl). Every Butane file is rendered on every
+    #     plan, so until they exist no plan succeeds -- not just this VM's.
+    #     Generate the key with `openssl rand -hex 32`, and keep "%" out of
+    #     the database URL: systemd reads it as a specifier
+    #
+    # Alerts, users and API keys all live in Postgres, so a taint loses
+    # nothing. SECRET_KEY signs login tokens; without a fixed one the image
+    # generates a new one per start and every session is dropped.
+    #
+    # ADMIN_PASSWORD is used only when the admin user does not exist yet.
+    # Changing it in Infisical afterwards has no effect: change it in the UI.
+    # Web logins go through Authelia; the basic-auth admin user is kept for
+    # API access. Self sign-up is off, so further users are added by the admin.
+    #
+    # --log-driver none keeps this container out of Dozzle. The image's
+    # entrypoint dumps its whole environment and prints the admin API key at
+    # every start, and dozzle-mcp above serves logs without auth. Output
+    # still reaches the journal (journalctl -u alerta) through the attached
+    # docker run.
+    - name: alerta.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Alerta alert console
+        After=docker.service versions-fetch.service
+        Requires=docker.service
+        Wants=versions-fetch.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        ExecStartPre=/usr/bin/docker pull $${ALERTA_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force alerta
+        ExecStart=/usr/bin/docker run --rm --name alerta \
+          --log-driver none \
+          -e DATABASE_URL=${alerta_database_url} \
+          -e SECRET_KEY=${alerta_secret_key} \
+          -e AUTH_REQUIRED=True \
+          -e AUTH_PROVIDER=openid \
+          -e OIDC_ISSUER_URL=https://auth.atlashorizon.net \
+          -e OAUTH2_CLIENT_ID=${alerta_oidc_client_id} \
+          -e OAUTH2_CLIENT_SECRET=${alerta_oidc_client_secret} \
+          -e SIGNUP_ENABLED=True \
+          -e ADMIN_USERS=admin \
+          -e ADMIN_PASSWORD=${alerta_admin_password} \
+          -p 8082:8080 \
+          $${ALERTA_IMAGE}
+        ExecStop=/usr/bin/docker stop alerta
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
 
 storage:
   files:
+    # Single source of truth for the agent list, shared by dozzle.service and
+    # dozzle-mcp.service. systemd EnvironmentFile format: no quoting, so the
+    # "|" and "," separators are taken literally.
+    - path: /etc/dozzle/agents.env
+      mode: 0644
+      contents:
+        inline: |
+          DOZZLE_REMOTE_AGENT=192.168.7.60:7007|oswald-runner|Goose,192.168.7.61:7007|ai|Goose,192.168.7.62:7007|oswl-connector|Goose,192.168.20.74:7007|Argo|Truenas,192.168.20.4:7007|Atalantia|Truenas,192.168.7.4:7007|Goldie|Truenas,192.168.20.217:7007|Laika|CallieCorner
     # Blocks until every agent named in DOZZLE_REMOTE_AGENT accepts a TCP
     # connection. The list is read from the environment, so adding or removing
     # an agent above needs no change here.

butane/oswald_runner.yaml.tftpl

@@ -25,9 +25,12 @@ systemd:
       contents: |
         [Unit]
         Description=Gitea Actions runner
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force oswald_runner
         ExecStart=/usr/bin/docker run --name oswald_runner \
@@ -36,26 +39,257 @@ systemd:
           -e GITEA_RUNNER_NAME=oswald_runner \
           -v /var/lib/oswald_runner:/data \
           -v /var/run/docker.sock:/var/run/docker.sock \
-          docker.io/gitea/runner:3
+          $${GITEA_RUNNER_IMAGE}
         ExecStop=/usr/bin/docker stop oswald_runner
         Restart=always
         RestartSec=5s
         [Install]
         WantedBy=multi-user.target
+    # n8n workflow automation, served publicly at n8n.atlashorizon.net.
+    #
+    # Prerequisites, all of which must exist before the VM is rebuilt:
+    #   - an "n8n" Postgres database and role on argo.lan.atlashorizon.net
+    #   - N8N_DB_PASSWORD and N8N_ENCRYPTION_KEY in Infisical (see
+    #     secrets.tmpl); generate the key with `openssl rand -hex 32`
+    #   - n8n.atlashorizon.net terminating TLS and proxying to this host's
+    #     :5678 (not managed here)
+    #
+    # Workflows, credentials and execution history all live in Postgres, so
+    # a taint loses nothing. The encryption key is what makes that true:
+    # without a fixed one n8n generates its own under /home/node/.n8n, and
+    # the credentials in Postgres become undecryptable the moment the VM
+    # disk goes. Never rotate it without re-entering every credential.
+    - name: n8n.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=n8n workflow automation
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
+        Requires=docker.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        ExecStartPre=/usr/bin/docker pull $${N8N_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force n8n
+        # /var/lib/n8n holds only what is left in /home/node/.n8n once the
+        # database and key are external: installed community nodes and
+        # n8n's own config file. Losing it to a taint means reinstalling
+        # community nodes, nothing more.
+        #
+        # N8N_PROXY_HOPS=1: requests arrive through the reverse proxy, so
+        # n8n takes the client IP from X-Forwarded-For (rate limiting and
+        # audit logs would otherwise see only the proxy).
+        ExecStart=/usr/bin/docker run --name n8n \
+          -e DB_TYPE=postgresdb \
+          -e DB_POSTGRESDB_HOST=argo.lan.atlashorizon.net \
+          -e DB_POSTGRESDB_PORT=5432 \
+          -e DB_POSTGRESDB_DATABASE=n8n \
+          -e DB_POSTGRESDB_USER=n8n \
+          -e DB_POSTGRESDB_PASSWORD=${n8n_db_password} \
+          -e N8N_ENCRYPTION_KEY=${n8n_encryption_key} \
+          -e N8N_HOST=n8n.atlashorizon.net \
+          -e N8N_PROTOCOL=https \
+          -e WEBHOOK_URL=https://n8n.atlashorizon.net/ \
+          -e N8N_PROXY_HOPS=1 \
+          -e N8N_DIAGNOSTICS_ENABLED=false \
+          -v /var/lib/n8n:/home/node/.n8n \
+          -p 5678:5678 \
+          $${N8N_IMAGE}
+        ExecStop=/usr/bin/docker stop n8n
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # CISA KEV RSS feed. Turns the CISA Known Exploited Vulnerabilities
+    # catalog into an RSS 2.0 feed (https://github.com/xsaveopt/cisa-kev-rss).
+    #
+    # Feed:   http://192.168.7.60:3000/rss
+    # Health: http://192.168.7.60:3000/health
+    #
+    # Stateless: the container refreshes from CISA every ten minutes and keeps
+    # nothing on disk. Moved here from its own VM (cisa-kev-rss, 192.168.7.64).
+    - name: cisa-kev-rss.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=CISA KEV RSS feed
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
+        Requires=docker.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        ExecStartPre=/usr/bin/docker pull $${CISA_KEV_RSS_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force cisa-kev-rss
+        ExecStart=/usr/bin/docker run --rm --name cisa-kev-rss \
+          -p 3000:3000 \
+          $${CISA_KEV_RSS_IMAGE}
+        ExecStop=/usr/bin/docker stop cisa-kev-rss
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # Domain Watchdog: RDAP-based domain expiry/change monitoring
+    # (https://github.com/maelgangloff/domain-watchdog), served at
+    # domainwatchdog.atlashorizon.net. Three containers on a private docker
+    # network: the web app on :8083, a messenger worker, and Valkey as the
+    # job queue and lock store.
+    #
+    # Prerequisites, all of which must exist before the VM is rebuilt:
+    #   - a "domainwatchdog" Postgres database and role on
+    #     argo.lan.atlashorizon.net; the schema is migrated on every start
+    #   - an OIDC client in Authelia with redirect URI
+    #     https://domainwatchdog.atlashorizon.net/login/oauth/check, scopes
+    #     openid/email/profile, and token_endpoint_auth_method
+    #     client_secret_post (the app sends the secret in the POST body)
+    #   - DOMAIN_WATCHDOG_DB_PASSWORD, DOMAIN_WATCHDOG_APP_SECRET,
+    #     DOMAIN_WATCHDOG_OIDC_CLIENT_ID and DOMAIN_WATCHDOG_OIDC_CLIENT_SECRET
+    #     in Infisical (see secrets.tmpl). Every Butane file is rendered on
+    #     every plan, so until they exist no plan succeeds. Generate the app
+    #     secret with `openssl rand -hex 32`; keep the DB password URL-safe
+    #     and free of "%" (it is spliced into DATABASE_URL, and systemd reads
+    #     "%" as a specifier)
+    #   - domainwatchdog.atlashorizon.net terminating TLS and proxying to this
+    #     host's :8083 with X-Forwarded-Proto/Host (not managed here)
+    #
+    # Users, watchlists and domain history all live in Postgres, so a taint
+    # loses nothing that matters. What does not survive it:
+    #   - queued RDAP jobs in Valkey; the scheduler re-queues them
+    #   - the JWT signing keys in /var/lib/domain-watchdog/jwt, which the
+    #     entrypoint regenerates when missing, so everyone logs in again.
+    #     They are on disk rather than in the container only so that a
+    #     plain restart does not do the same.
+    #
+    # Login is Authelia-only: registration is off, and the first OIDC login
+    # for an email creates that user, so Authelia's access rules for the
+    # client decide who gets in.
+    - name: domain-watchdog-valkey.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Domain Watchdog Valkey
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
+        Requires=docker.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        ExecStartPre=/usr/bin/docker pull $${DOMAIN_WATCHDOG_VALKEY_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force domain-watchdog-valkey
+        # Every Domain Watchdog unit tries to create the network, so start
+        # order does not matter; "-" because all but the first will fail.
+        ExecStartPre=-/usr/bin/docker network create domain-watchdog
+        # No volume and no port: the queue is disposable and only the other
+        # two containers reach it, by name, over the domain-watchdog network.
+        ExecStart=/usr/bin/docker run --rm --name domain-watchdog-valkey \
+          --network domain-watchdog \
+          --network-alias valkey \
+          $${DOMAIN_WATCHDOG_VALKEY_IMAGE}
+        ExecStop=/usr/bin/docker stop domain-watchdog-valkey
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    - name: domain-watchdog.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Domain Watchdog web app
+        After=docker.service domain-watchdog-valkey.service versions-fetch.service
+        Requires=docker.service
+        Wants=domain-watchdog-valkey.service versions-fetch.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        # Secrets are unit environment rather than lines in app.env, which
+        # is world-readable; docker run forwards them by bare name below.
+        Environment="APP_SECRET=${domain_watchdog_app_secret}"
+        Environment="DATABASE_URL=postgresql://domainwatchdog:${domain_watchdog_db_password}@argo.lan.atlashorizon.net:5432/domainwatchdog?serverVersion=16&charset=utf8"
+        Environment="OAUTH_CLIENT_ID=${domain_watchdog_oidc_client_id}"
+        Environment="OAUTH_CLIENT_SECRET=${domain_watchdog_oidc_client_secret}"
+        ExecStartPre=/usr/bin/docker pull $${DOMAIN_WATCHDOG_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force domain-watchdog
+        ExecStartPre=-/usr/bin/docker network create domain-watchdog
+        # The entrypoint runs DB migrations and generates the JWT keypair
+        # into /app/config/jwt if it is missing, then starts FrankenPHP on
+        # :80 (plain HTTP: SERVER_NAME=:80 in app.env, TLS is the proxy's).
+        ExecStart=/usr/bin/docker run --rm --name domain-watchdog \
+          --network domain-watchdog \
+          --env-file /etc/domain-watchdog/app.env \
+          -e APP_SECRET \
+          -e DATABASE_URL \
+          -e OAUTH_CLIENT_ID \
+          -e OAUTH_CLIENT_SECRET \
+          -v /var/lib/domain-watchdog/jwt:/app/config/jwt \
+          -v /etc/domain-watchdog/content:/app/public/content:ro \
+          -p 8083:80 \
+          $${DOMAIN_WATCHDOG_IMAGE}
+        ExecStop=/usr/bin/docker stop domain-watchdog
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # Runs the queued RDAP lookups and notifications. Same image, so the
+    # entrypoint also runs migrations here; ordering after the web app keeps
+    # the two from racing on a fresh database. It exits every hour
+    # (--time-limit, as upstream recommends against PHP memory creep) and
+    # Restart=always brings it straight back.
+    - name: domain-watchdog-worker.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Domain Watchdog worker
+        After=docker.service domain-watchdog-valkey.service domain-watchdog.service versions-fetch.service
+        Requires=docker.service
+        Wants=domain-watchdog-valkey.service versions-fetch.service
+        [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
+        TimeoutStartSec=0
+        Environment="APP_SECRET=${domain_watchdog_app_secret}"
+        Environment="DATABASE_URL=postgresql://domainwatchdog:${domain_watchdog_db_password}@argo.lan.atlashorizon.net:5432/domainwatchdog?serverVersion=16&charset=utf8"
+        ExecStartPre=/usr/bin/docker pull $${DOMAIN_WATCHDOG_IMAGE}
+        ExecStartPre=-/usr/bin/docker rm --force domain-watchdog-worker
+        ExecStartPre=-/usr/bin/docker network create domain-watchdog
+        # --no-healthcheck: the image's HEALTHCHECK probes FrankenPHP's
+        # metrics on :2019, which only the web app runs, so the worker would
+        # always show unhealthy (upstream's compose disables it the same way).
+        ExecStart=/usr/bin/docker run --rm --name domain-watchdog-worker \
+          --no-healthcheck \
+          --network domain-watchdog \
+          --env-file /etc/domain-watchdog/app.env \
+          -e APP_SECRET \
+          -e DATABASE_URL \
+          -e MESSENGER_CONSUMER_NAME=worker \
+          $${DOMAIN_WATCHDOG_IMAGE} \
+          php /app/bin/console messenger:consume --all --time-limit=3600 -vv
+        ExecStop=/usr/bin/docker stop domain-watchdog-worker
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
     - name: dozzle-agent.service
       enabled: true
       contents: |
         [Unit]
         Description=Dozzle Agent
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force dozzle-agent
         ExecStart=/usr/bin/docker run --name dozzle-agent \
           -v /var/run/docker.sock:/var/run/docker.sock \
           -p 7007:7007 \
-          amir20/dozzle:v11.1.1 agent
+          $${DOZZLE_IMAGE} agent
         ExecStop=/usr/bin/docker stop dozzle-agent
         Restart=always
         RestartSec=5s
@@ -66,3 +300,85 @@ storage:
   directories:
     - path: /var/lib/oswald_runner
       mode: 0755
+    # The n8n image runs as the "node" user (uid 1000), which needs to own
+    # its data directory.
+    - path: /var/lib/n8n
+      mode: 0700
+      user:
+        id: 1000
+      group:
+        id: 1000
+    # Domain Watchdog's JWT keypair; the container runs as root.
+    - path: /var/lib/domain-watchdog/jwt
+      mode: 0700
+  files:
+    # Non-secret Domain Watchdog settings, shared by the web app and worker
+    # units (docker --env-file format: no quoting, values taken literally).
+    - path: /etc/domain-watchdog/app.env
+      mode: 0644
+      contents:
+        inline: |
+          APP_ENV=prod
+          SERVER_NAME=:80
+          HTTP_SECURE_COOKIE=true
+          SYMFONY_TRUSTED_PROXIES=private_ranges
+          SYMFONY_TRUSTED_HEADERS=x-forwarded-for,x-forwarded-host,x-forwarded-proto,x-forwarded-port
+          LOCK_DSN=redis://valkey:6379
+          MESSENGER_ASYNC_TRANSPORT_DSN=redis://valkey:6379/messages
+          MESSENGER_RDAP_LOW_TRANSPORT_DSN=redis://valkey:6379/messages-rdap-low
+          MESSENGER_RDAP_HIGH_TRANSPORT_DSN=redis://valkey:6379/messages-rdap-high
+          REGISTRATION_ENABLED=false
+          REGISTRATION_VERIFY_EMAIL=false
+          OAUTH_AUTHORIZATION_URL=https://auth.atlashorizon.net/api/oidc/authorization
+          OAUTH_TOKEN_URL=https://auth.atlashorizon.net/api/oidc/token
+          OAUTH_USERINFO_URL=https://auth.atlashorizon.net/api/oidc/userinfo
+          OAUTH_SCOPE=openid email profile
+          SSO_AUTO_REDIRECT=true
+          MAILER_DSN=null://null
+    # Markdown pages the frontend fetches from /content/<name>; a missing one
+    # renders as "Please create the /public/content/<name> file." Mounted
+    # read-only into the web app. ads.md is shown on the login page and is
+    # left empty on purpose (an empty file renders as nothing).
+    - path: /etc/domain-watchdog/content/home.md
+      mode: 0644
+      contents:
+        inline: |
+          # Domain Watchdog
+
+          Tracks our domains through RDAP and alerts on expiry, transfers and
+          other registry changes.
+
+          Add domains under **Tracking > Watchlists**, and attach a
+          notification connector to a watchlist to be alerted.
+    - path: /etc/domain-watchdog/content/faq.md
+      mode: 0644
+      contents:
+        inline: |
+          # FAQ
+
+          **Who can log in?** Anyone allowed through Authelia for this app.
+          Your account is created on first login.
+
+          **How often are domains checked?** The worker refreshes watched
+          domains from RDAP on a schedule; domains close to expiry are
+          checked more often.
+    - path: /etc/domain-watchdog/content/tos.md
+      mode: 0644
+      contents:
+        inline: |
+          # Terms of use
+
+          Private instance for Atlas Horizon use only.
+    - path: /etc/domain-watchdog/content/privacy.md
+      mode: 0644
+      contents:
+        inline: |
+          # Privacy
+
+          Stores your email address (from Authelia) and your watchlists.
+          Nothing is shared with third parties; RDAP lookups go directly to
+          the registries.
+    - path: /etc/domain-watchdog/content/ads.md
+      mode: 0644
+      contents:
+        inline: ""

butane/oswl_connector.yaml.tftpl

@@ -20,10 +20,12 @@ systemd:
       contents: |
         [Unit]
         Description=Tailscale node (headscale subnet router + exit node)
-        After=docker.service network-online.target
+        After=docker.service network-online.target versions-fetch.service
         Requires=docker.service
-        Wants=network-online.target
+        Wants=network-online.target versions-fetch.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         # None of these are loaded by default on a stock Flatcar boot, and the
         # container cannot load them itself: it has NET_ADMIN (enough to write
@@ -49,7 +51,7 @@ systemd:
           -e TS_ACCEPT_DNS=false \
           -e "TS_EXTRA_ARGS=--login-server=https://hs.atlashorizon.net --advertise-exit-node" \
           -e TS_USERSPACE=false \
-          docker.io/tailscale/tailscale:stable
+          $${TAILSCALE_IMAGE}
         ExecStop=/usr/bin/docker stop tailscaled
         Restart=always
         RestartSec=5s
@@ -60,15 +62,18 @@ systemd:
       contents: |
         [Unit]
         Description=Dozzle Agent
-        After=docker.service
+        After=docker.service versions-fetch.service
+        Wants=versions-fetch.service
         Requires=docker.service
         [Service]
+        # Image tag comes from versions.env; see butane/versions-sync.yaml.
+        EnvironmentFile=/etc/versions.env
         TimeoutStartSec=0
         ExecStartPre=-/usr/bin/docker rm --force dozzle-agent
         ExecStart=/usr/bin/docker run --name dozzle-agent \
           -v /var/run/docker.sock:/var/run/docker.sock \
           -p 7007:7007 \
-          amir20/dozzle:v11.1.1 agent
+          $${DOZZLE_IMAGE} agent
         ExecStop=/usr/bin/docker stop dozzle-agent
         Restart=always
         RestartSec=5s

butane/versions-sync.yaml

@@ -0,0 +1,113 @@
+variant: flatcar
+version: 1.0.0
+
+# Shared by every Flatcar VM: main.tf passes this file as each VM's inline
+# butane snippet. Loaded with file(), not templatefile(), so it is not a
+# template -- ${...} below reaches the VM as written.
+#
+# Keeps /etc/versions.env in sync with versions.env at the root of
+# AtlasHorizon/compose, which holds the image tag of every container on the
+# Flatcar VMs. A version bump merged to the branch below reaches each VM
+# within ~10 minutes with no plan or rebuild; see fetch-versions.sh for how
+# the affected units are found and restarted.
+#
+# To opt a container unit in, in its butane file:
+#   [Unit]     After=... versions-fetch.service
+#              Wants=versions-fetch.service
+#   [Service]  EnvironmentFile=/etc/versions.env
+# and use the variable, in braces, wherever the image goes (the docker pull
+# and docker run arguments). In a .tftpl file the braces need a doubled
+# dollar sign so templatefile() leaves them alone.
+
+systemd:
+  units:
+    # Not enabled itself: the container units pull it in at boot (Wants= and
+    # After=), so they start on the freshest copy, and the timer re-runs it.
+    # A failed fetch keeps the last good copy, so a Gitea outage only delays
+    # updates -- except on a brand-new VM, whose units cannot start until the
+    # first fetch succeeds. Restart= keeps retrying that first fetch.
+    - name: versions-fetch.service
+      contents: |
+        [Unit]
+        Description=Fetch image versions from AtlasHorizon/compose
+        Wants=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        # Override with a drop-in to test unreleased versions on one VM.
+        Environment=VERSIONS_BRANCH=main
+        ExecStart=/opt/versions/fetch-versions.sh
+        Restart=on-failure
+        RestartSec=30s
+    - name: versions-fetch.timer
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Periodically re-fetch image versions
+        [Timer]
+        OnBootSec=10min
+        OnUnitActiveSec=10min
+        RandomizedDelaySec=1min
+        [Install]
+        WantedBy=timers.target
+
+storage:
+  files:
+    - path: /opt/versions/fetch-versions.sh
+      mode: 0755
+      contents:
+        inline: |
+          #!/usr/bin/bash
+          # Fetches versions.env and restarts the units that use a variable
+          # whose value changed. A unit "uses" a variable when its unit file
+          # contains it in braces (the docker pull/run image arguments), so
+          # adding a container needs no change here.
+          set -euo pipefail
+
+          url="https://git.atlashorizon.net/AtlasHorizon/compose/raw/branch/$VERSIONS_BRANCH/versions.env"
+          dest=/etc/versions.env
+
+          tmp=$(mktemp /etc/.versions.env.XXXXXX)
+          trap 'rm -f "$tmp"' EXIT
+
+          curl -fsS --max-time 30 -o "$tmp" "$url"
+
+          # Only comments, blank lines and VAR=value lines are accepted, and
+          # at least one *_IMAGE line must be present, so an error page or a
+          # truncated download never replaces the last good copy.
+          if grep -qvE '^(#.*|[A-Z0-9_]+=[^[:space:]]+|)$' "$tmp" ||
+             ! grep -qE '^[A-Z0-9_]+_IMAGE=' "$tmp"; then
+            echo "refusing malformed versions.env from $url" >&2
+            exit 1
+          fi
+
+          if [ -f "$dest" ] && cmp -s "$tmp" "$dest"; then
+            exit 0
+          fi
+
+          # With no previous copy this is a brand-new VM: its units are waiting
+          # on (or already failed without) this file, so start them outright.
+          # Otherwise only units that are running get restarted, so a unit
+          # stopped by hand stays stopped.
+          if [ -f "$dest" ]; then
+            action=try-restart
+            changed=$(diff <(grep -E '^[A-Z0-9_]+=' "$dest" | sort) \
+                           <(grep -E '^[A-Z0-9_]+=' "$tmp" | sort) |
+                      sed -nE 's/^[<>] ([A-Z0-9_]+)=.*/\1/p' | sort -u || true)
+          else
+            action=restart
+            changed=$(sed -nE 's/^([A-Z0-9_]+)=.*/\1/p' "$tmp")
+          fi
+
+          chmod 0644 "$tmp"
+          mv -f "$tmp" "$dest"
+
+          for var in $changed; do
+            echo "$var changed"
+            pattern=$(printf '${%s}' "$var")
+            # --no-block: the units are ordered after this one, so waiting on
+            # their jobs from inside it would deadlock at boot.
+            for unit in $(grep -lF "$pattern" /etc/systemd/system/*.service || true); do
+              systemctl --no-block "$action" "$(basename "$unit")"
+            done
+          done

main.tf

@@ -36,6 +36,20 @@ locals {
 # provider alias in providers.tf and a module block here. Everything else
 # (node contents, common config) lives in the parent repo's terraform.tfvars.
 
+# butane/versions-sync.yaml goes onto every Flatcar VM through its inline
+# butane field (the module merges it with butane_file). Inline snippets are
+# not templated, so the file is read as-is. No VM sets butane in tfvars; one
+# that did would replace this snippet, not add to it, and lose version sync.
+locals {
+  flatcar_common_butane = file("${path.module}/butane/versions-sync.yaml")
+  flatcar_with_common = {
+    for hv in ["proxmox", "goose"] : hv => {
+      for name, vm in try(var.nodes[hv].flatcar, {}) :
+      name => merge(vm, { butane = coalesce(vm.butane, local.flatcar_common_butane) })
+    }
+  }
+}
+
 module "proxmox" {
   source = "./opentofu/modules/proxmox_node"
 
@@ -43,7 +57,7 @@ module "proxmox" {
   storage              = var.hypervisors["proxmox"].storage
   lxc                  = try(var.nodes["proxmox"].lxc, {})
   machines             = try(var.nodes["proxmox"].machines, {})
-  flatcar              = try(var.nodes["proxmox"].flatcar, {})
+  flatcar              = local.flatcar_with_common["proxmox"]
   search_domain        = local.node_common.search_domain
   dns_nameservers      = local.node_common.dns_nameservers
   ssh_key              = local.node_common.ssh_key
@@ -68,7 +82,7 @@ module "goose" {
   storage              = var.hypervisors["goose"].storage
   lxc                  = try(var.nodes["goose"].lxc, {})
   machines             = try(var.nodes["goose"].machines, {})
-  flatcar              = try(var.nodes["goose"].flatcar, {})
+  flatcar              = local.flatcar_with_common["goose"]
   search_domain        = local.node_common.search_domain
   dns_nameservers      = local.node_common.dns_nameservers
   ssh_key              = local.node_common.ssh_key

secrets.tmpl

@@ -14,9 +14,30 @@
   {{- if eq $secret.Key "LITELLM_DATABASE_URL" }}"litellm_database_url": "{{$secret.Value}}",{{ end }}
   {{- if eq $secret.Key "LITELLM_MASTER_KEY" }}"litellm_master_key": "{{$secret.Value}}",{{ end }}
   {{- if eq $secret.Key "LITELLM_SALT_KEY" }}"litellm_salt_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENCODE_SERVER_PASSWORD" }}"opencode_server_password": "{{$secret.Value}}",{{ end }}
   {{- if eq $secret.Key "HEADSCALE_AUTHKEY" }}"headscale_authkey": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENWEBUI_DATABASE_URL" }}"openwebui_database_url": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENWEBUI_SECRET_KEY" }}"openwebui_secret_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENWEBUI_S3_ACCESS_KEY" }}"openwebui_s3_access_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENWEBUI_S3_SECRET_KEY" }}"openwebui_s3_secret_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENWEBUI_OAUTH_CLIENT_ID" }}"openwebui_oauth_client_id": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENWEBUI_OAUTH_CLIENT_SECRET" }}"openwebui_oauth_client_secret": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENWEBUI_OIDC_PROVIDER_URL" }}"openwebui_oidc_provider_url": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENWEBUI_TERMINAL_API_KEY" }}"openwebui_terminal_api_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "OPENWEBUI_ADMIN_API_KEY" }}"openwebui_admin_api_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "N8N_DB_PASSWORD" }}"n8n_db_password": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "N8N_ENCRYPTION_KEY" }}"n8n_encryption_key": "{{$secret.Value}}",{{ end }}
   {{- if eq $secret.Key "DOZZLE_OIDC_CLIENT_ID" }}"dozzle_oidc_client_id": "{{$secret.Value}}",{{ end }}
   {{- if eq $secret.Key "DOZZLE_OIDC_CLIENT_SECRET" }}"dozzle_oidc_client_secret": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_DATABASE_URL" }}"alerta_database_url": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_SECRET_KEY" }}"alerta_secret_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_ADMIN_PASSWORD" }}"alerta_admin_password": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_OIDC_CLIENT_ID" }}"alerta_oidc_client_id": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "ALERTA_OIDC_CLIENT_SECRET" }}"alerta_oidc_client_secret": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "DOMAIN_WATCHDOG_DB_PASSWORD" }}"domain_watchdog_db_password": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "DOMAIN_WATCHDOG_APP_SECRET" }}"domain_watchdog_app_secret": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "DOMAIN_WATCHDOG_OIDC_CLIENT_ID" }}"domain_watchdog_oidc_client_id": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "DOMAIN_WATCHDOG_OIDC_CLIENT_SECRET" }}"domain_watchdog_oidc_client_secret": "{{$secret.Value}}",{{ end }}
   {{- end }}
   "_sentinel": "1"
 }

terraform.tfvars

@@ -203,13 +203,10 @@ nodes = {
         vmid             = 210
         ip               = "192.168.7.60"
         vlan             = 7
-        memory           = 4096
+        memory           = 6144
         cores            = 4
         disk_size        = 32
         qemu_guest_agent = false
-        # Rendered with templatefile(): ${gitea_runner_token} comes from
-        # Infisical (GITEA_RUNNER_TOKEN in secrets.tmpl), so the token is
-        # never committed.
         butane_file = "butane/oswald_runner.yaml.tftpl"
       }
       "ai" = {
@@ -217,15 +214,12 @@ nodes = {
         vmid             = 211
         ip               = "192.168.7.61"
         vlan             = 7
-        memory           = 4096
-        cores            = 2
+        memory           = 8192
+        cores            = 4
         disk_size        = 32
         qemu_guest_agent = false
         butane_file = "butane/ai.yaml.tftpl"
       }
-      # Headscale subnet router / exit node. Registers as an ephemeral node,
-      # so the preauth key must be --ephemeral --reusable; the routes and the
-      # exit node still need approving on the headscale side once it appears.
       "oswl-connector" = {
         hostname         = "oswl-connector"
         vmid             = 212
@@ -237,33 +231,17 @@ nodes = {
         qemu_guest_agent = false
         butane_file      = "butane/oswl_connector.yaml.tftpl"
       }
-      # Dozzle log viewer hub. Pulls container logs from the dozzle agents on
-      # the other Flatcar VMs (port 7007) plus its own local docker socket.
-      # Keeps no state on disk, so it is safe to rebuild at any time.
       "monitor" = {
         hostname         = "monitor"
         vmid             = 213
         ip               = "192.168.7.63"
         vlan             = 7
-        memory           = 2048
+        memory           = 4096
         cores            = 2
         disk_size        = 32
         qemu_guest_agent = false
         butane_file      = "butane/monitor.yaml.tftpl"
       }
-      # CISA Known Exploited Vulnerabilities catalog as an RSS feed.
-      # Stateless container; feed at :3000/rss.
-      "cisa-kev-rss" = {
-        hostname         = "cisa-kev-rss"
-        vmid             = 214
-        ip               = "192.168.7.64"
-        vlan             = 7
-        memory           = 1024
-        cores            = 1
-        disk_size        = 16
-        qemu_guest_agent = false
-        butane_file      = "butane/cisa_kev_rss.yaml"
-      }
     }
     machines = {
       "grist" = {
@@ -278,7 +256,7 @@ nodes = {
         agent      = 1
         memory     = 4096
         disk_size  = "32"
-        balloon    = 1
+        balloon    = 1024
         cpu_type   = "kvm64"
         cores      = 4
         sockets    = 1