oswald-infra/butane/monitor.yaml.tftpl

variant: flatcar
version: 1.0.0

# Dozzle log viewer hub.
#
# Rendered with templatefile(): the OIDC client id and secret come from
# Infisical (DOZZLE_OIDC_CLIENT_ID / DOZZLE_OIDC_CLIENT_SECRET in
# secrets.tmpl), so neither is committed. The secret does still land in
# plaintext in the rendered Ignition snippet on the node, and in
# "systemctl show dozzle.service" -- same caveat as headscale_authkey.
#
# Aggregates container logs from the dozzle agents running on the other Flatcar
# VMs (see the dozzle-agent units in the other butane configs) and, via the
# local socket, from this host's own containers. Reachable on :8080.
#
# Also runs Alerta on :8082; see alerta.service below.
#
# No state is kept on disk: /data is a tmpfs, so the container is fully
# disposable and there is nothing to back up. That is also why simple auth
# (users.yml) is not used -- it would need a persistent /data.
#
# Hub <-> agent traffic is TLS, using the certificate pair baked into the
# dozzle image. Every hub and agent on the same image version shares it, so
# nothing needs mounting here -- which is also why the agents must stay pinned
# to the same tag as the hub (DOZZLE_IMAGE in versions.env). To move to a
# custom pair later, generate one with (writes into $PWD, using the tag from
# DOZZLE_IMAGE):
#
#   docker run --rm -v "$PWD":/out amir20/dozzle:<tag> generate-certs \
#     --cert-out /out/dozzle_cert.pem --key-out /out/dozzle_key.pem
#
# ...then mount it at /dozzle_cert.pem and /dozzle_key.pem (read-only) on the
# hub and on every agent. Keep the key out of this file: it would end up in
# plaintext in the rendered Ignition snippet on the node's storage.

systemd:
  units:
    - name: dozzle.service
      enabled: true
      contents: |
        [Unit]
        Description=Dozzle log viewer hub
        After=docker.service versions-fetch.service
        Requires=docker.service
        Wants=versions-fetch.service
        [Service]
        # Image tag comes from versions.env; see butane/versions-sync.yaml.
        EnvironmentFile=/etc/versions.env
        TimeoutStartSec=0
        # Kept as unit environment rather than inline -e values so that
        # wait-for-agents.sh can read the agent list from the same place the
        # container does. docker run forwards them by bare name below.
        Environment="DOZZLE_HOSTNAME=Oswald Logs"
        # The agent list lives in a file because dozzle-mcp.service needs the
        # identical list; keeping it inline in both units would drift.
        EnvironmentFile=/etc/dozzle/agents.env
        # OIDC via Authelia. Dozzle builds its redirect from the incoming
        # request, so the proxy in front must pass the real Host and
        # X-Forwarded-Proto; the URI to register in Authelia is
        # https://dozzle.oswaldtheotter.net/api/auth/callback
        # Authelia publishes no "roles"/"dozzle_roles" claim, only "groups",
        # so the groups claim is requested as an extra scope and pointed at
        # directly -- otherwise Dozzle finds no roles and nobody gets admin.
        Environment="DOZZLE_AUTH_PROVIDER=oidc"
        Environment="DOZZLE_AUTH_OIDC_ISSUER=https://auth.atlashorizon.net"
        Environment="DOZZLE_AUTH_OIDC_CLIENT_ID=${dozzle_oidc_client_id}"
        Environment="DOZZLE_AUTH_OIDC_CLIENT_SECRET=${dozzle_oidc_client_secret}"
        Environment="DOZZLE_AUTH_OIDC_NAME=AtlasHorizon"
        Environment="DOZZLE_AUTH_OIDC_SCOPES=dozzle_roles,picture"
        Environment="DOZZLE_AUTH_OIDC_ROLES_CLAIM=dozzle_roles"
        # Pulled up front so a slow first pull is not mistaken for a hung unit.
        ExecStartPre=/usr/bin/docker pull $${DOZZLE_IMAGE}
        # Leading "-" so a missing container is not a failure.
        ExecStartPre=-/usr/bin/docker rm --force dozzle
        # If the hub starts before the agents are listening, the failed initial
        # handshake leaves a permanently-offline placeholder host in the UI
        # alongside the real one once it connects. Bounded and "-" prefixed, so
        # an agent that is genuinely gone delays the hub but never blocks it.
        ExecStartPre=-/usr/bin/timeout 180 /opt/dozzle/wait-for-agents.sh
        # Foreground (no -d) so systemd supervises the container process.
        ExecStart=/usr/bin/docker run --rm --name dozzle \
          -v /var/run/docker.sock:/var/run/docker.sock:ro \
          --tmpfs /data \
          -p 8080:8080 \
          -e DOZZLE_HOSTNAME \
          -e DOZZLE_REMOTE_AGENT \
          -e DOZZLE_AUTH_PROVIDER \
          -e DOZZLE_AUTH_OIDC_ISSUER \
          -e DOZZLE_AUTH_OIDC_CLIENT_ID \
          -e DOZZLE_AUTH_OIDC_CLIENT_SECRET \
          -e DOZZLE_AUTH_OIDC_NAME \
          -e DOZZLE_AUTH_OIDC_SCOPES \
          -e DOZZLE_AUTH_OIDC_ROLES_CLAIM \
          $${DOZZLE_IMAGE}
        ExecStop=/usr/bin/docker stop dozzle
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target
    # Second Dozzle, dedicated to MCP on :8081.
    #
    # It exists because the two requirements cannot be met by one instance:
    # the oidc provider is the only one that lets Authelia own the user list,
    # but it offers no non-interactive login, so an MCP client can never
    # authenticate against it. This instance therefore runs with no auth at
    # all (DOZZLE_AUTH_PROVIDER unset) and is reached only through the
    # Cloudflare tunnel, which is where access control has to live.
    #
    # ANYTHING that can open :8081 gets unauthenticated read access to every
    # log on every agent. Do not publish this port beyond the tunnel, and put
    # a Cloudflare Access policy (service token) on its hostname.
    - name: dozzle-mcp.service
      enabled: true
      contents: |
        [Unit]
        Description=Dozzle MCP endpoint
        After=docker.service versions-fetch.service
        Requires=docker.service
        Wants=versions-fetch.service
        [Service]
        # Image tag comes from versions.env; see butane/versions-sync.yaml.
        EnvironmentFile=/etc/versions.env
        TimeoutStartSec=0
        # Same agents as the hub, same file, so the two cannot drift.
        EnvironmentFile=/etc/dozzle/agents.env
        Environment="DOZZLE_HOSTNAME=Oswald Logs (MCP)"
        Environment="DOZZLE_ENABLE_MCP=true"
        ExecStartPre=/usr/bin/docker pull $${DOZZLE_IMAGE}
        ExecStartPre=-/usr/bin/docker rm --force dozzle-mcp
        ExecStartPre=-/usr/bin/timeout 180 /opt/dozzle/wait-for-agents.sh
        ExecStart=/usr/bin/docker run --rm --name dozzle-mcp \
          -v /var/run/docker.sock:/var/run/docker.sock:ro \
          --tmpfs /data \
          -p 8081:8080 \
          -e DOZZLE_HOSTNAME \
          -e DOZZLE_REMOTE_AGENT \
          -e DOZZLE_ENABLE_MCP \
          $${DOZZLE_IMAGE}
        ExecStop=/usr/bin/docker stop dozzle-mcp
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target
    # Alerta alert console (API + web UI) on :8082.
    #
    # Prerequisites, all of which must exist before the VM is rebuilt:
    #   - an "alerta" Postgres database and role; the schema is created on
    #     first start
    #   - an OIDC client for Alerta in Authelia, with the Alerta URL as a
    #     redirect URI and token_endpoint_auth_method client_secret_basic
    #   - ALERTA_DATABASE_URL, ALERTA_SECRET_KEY, ALERTA_ADMIN_PASSWORD,
    #     ALERTA_OIDC_CLIENT_ID and ALERTA_OIDC_CLIENT_SECRET in
    #     Infisical (see secrets.tmpl). Every Butane file is rendered on every
    #     plan, so until they exist no plan succeeds -- not just this VM's.
    #     Generate the key with `openssl rand -hex 32`, and keep "%" out of
    #     the database URL: systemd reads it as a specifier
    #
    # Alerts, users and API keys all live in Postgres, so a taint loses
    # nothing. SECRET_KEY signs login tokens; without a fixed one the image
    # generates a new one per start and every session is dropped.
    #
    # ADMIN_PASSWORD is used only when the admin user does not exist yet.
    # Changing it in Infisical afterwards has no effect: change it in the UI.
    # Web logins go through Authelia; the basic-auth admin user is kept for
    # API access. Self sign-up is off, so further users are added by the admin.
    #
    # --log-driver none keeps this container out of Dozzle. The image's
    # entrypoint dumps its whole environment and prints the admin API key at
    # every start, and dozzle-mcp above serves logs without auth. Output
    # still reaches the journal (journalctl -u alerta) through the attached
    # docker run.
    - name: alerta.service
      enabled: true
      contents: |
        [Unit]
        Description=Alerta alert console
        After=docker.service versions-fetch.service
        Requires=docker.service
        Wants=versions-fetch.service
        [Service]
        # Image tag comes from versions.env; see butane/versions-sync.yaml.
        EnvironmentFile=/etc/versions.env
        TimeoutStartSec=0
        ExecStartPre=/usr/bin/docker pull $${ALERTA_IMAGE}
        ExecStartPre=-/usr/bin/docker rm --force alerta
        ExecStart=/usr/bin/docker run --rm --name alerta \
          --log-driver none \
          -e DATABASE_URL=${alerta_database_url} \
          -e SECRET_KEY=${alerta_secret_key} \
          -e AUTH_REQUIRED=True \
          -e AUTH_PROVIDER=openid \
          -e OIDC_ISSUER_URL=https://auth.atlashorizon.net \
          -e OAUTH2_CLIENT_ID=${alerta_oidc_client_id} \
          -e OAUTH2_CLIENT_SECRET=${alerta_oidc_client_secret} \
          -e SIGNUP_ENABLED=True \
          -e ADMIN_USERS=admin \
          -e ADMIN_PASSWORD=${alerta_admin_password} \
          -p 8082:8080 \
          $${ALERTA_IMAGE}
        ExecStop=/usr/bin/docker stop alerta
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target

storage:
  files:
    # Single source of truth for the agent list, shared by dozzle.service and
    # dozzle-mcp.service. systemd EnvironmentFile format: no quoting, so the
    # "|" and "," separators are taken literally.
    - path: /etc/dozzle/agents.env
      mode: 0644
      contents:
        inline: |
          DOZZLE_REMOTE_AGENT=192.168.7.60:7007|oswald-runner|Goose,192.168.7.61:7007|ai|Goose,192.168.7.62:7007|oswl-connector|Goose,192.168.20.74:7007|Argo|Truenas,192.168.20.4:7007|Atalantia|Truenas,192.168.7.4:7007|Goldie|Truenas,192.168.20.217:7007|Laika|CallieCorner
    # Blocks until every agent named in DOZZLE_REMOTE_AGENT accepts a TCP
    # connection. The list is read from the environment, so adding or removing
    # an agent above needs no change here.
    - path: /opt/dozzle/wait-for-agents.sh
      mode: 0755
      contents:
        inline: |
          #!/usr/bin/bash
          # Note: /dev/tcp is unavailable -- Flatcar's bash is built without
          # net redirections -- so reachability is probed with curl. The agent
          # speaks gRPC over TLS, so any completed connection answers with a
          # protocol error; only exit 7 (refused) and 28 (timeout) mean the
          # agent is not up yet.
          if [ -z "$DOZZLE_REMOTE_AGENT" ]; then
            exit 0
          fi

          for entry in $(echo "$DOZZLE_REMOTE_AGENT" | tr ',' ' '); do
            # Strip the |name|group suffix; what is left is host:port.
            hostport=$(echo "$entry" | cut -d'|' -f1)
            while true; do
              curl -sk -o /dev/null --max-time 3 "https://$hostport"
              rc=$?
              case "$rc" in
                7|28)
                  echo "waiting for dozzle agent $hostport"
                  sleep 2
                  ;;
                *)
                  break
                  ;;
              esac
            done
          done