oswald-infra/butane/ai.yaml.tftpl

variant: flatcar
version: 1.0.0

systemd:
  units:
    - name: gitea-mcp.service
      enabled: true
      contents: |
        [Unit]
        Description=Gitea MCP Server
        After=docker.service
        Requires=docker.service
        [Service]
        TimeoutStartSec=0
        ExecStartPre=-/usr/bin/docker rm --force gitea-mcp
        # The image ships a broken HEALTHCHECK: it is CMD-SHELL form but the
        # image is distroless, so every probe fails on a missing /bin/sh and
        # the container is reported unhealthy while serving fine. Exec-form
        # health tests cannot be set from docker run, so the check is dropped;
        # "/app/gitea-mcp -healthcheck" still works via docker exec.
        ExecStart=/usr/bin/docker run --name gitea-mcp --no-healthcheck \
          -e GITEA_HOST=https://git.atlashorizon.net \
          -p 8080:8080 \
          docker.gitea.com/gitea-mcp-server \
          /app/gitea-mcp -t http --port 8080
        ExecStop=/usr/bin/docker stop gitea-mcp
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target
    - name: litellm.service
      enabled: true
      contents: |
        [Unit]
        Description=LiteLLM proxy
        After=docker.service
        Requires=docker.service
        [Service]
        TimeoutStartSec=0
        ExecStartPre=-/usr/bin/docker rm --force litellm
        ExecStart=/usr/bin/docker run --name litellm \
          -e DATABASE_URL=${litellm_database_url} \
          -e STORE_MODEL_IN_DB=True \
          -e LITELLM_MASTER_KEY=${litellm_master_key} \
          -e LITELLM_SALT_KEY=${litellm_salt_key} \
          -e LITELLM_LOG=ERROR \
          -e JSON_LOGS=True \
          -v /etc/litellm/config.yaml:/app/config.yaml:ro \
          -p 4000:4000 \
          docker.litellm.ai/berriai/litellm:main-stable \
          --config /app/config.yaml --port 4000
        ExecStop=/usr/bin/docker stop litellm
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target
    # Open WebUI, with Qdrant alongside it for document embeddings.
    #
    # Prerequisites, all of which must exist before the VM is rebuilt:
    #   - Postgres database (plain; no pgvector extension needed -- embeddings
    #     go to Qdrant, and Postgres only holds users, chats and settings)
    #   - openwebui bucket in Garage, plus a key with r/w on it
    #   - an embedding model in LiteLLM matching RAG_EMBEDDING_MODEL below
    #   - an OIDC client at the provider, with the callback URI registered as
    #     https://llm.atlashorizon.net/oauth/oidc/callback
    #   - llm.atlashorizon.net terminating TLS and proxying to this host's
    #     :3000 (not managed here -- see WEBUI_URL below)
    #   - the OPENWEBUI_* secrets in Infisical (see secrets.tmpl)
    #
    # After a taint: the vector store is empty and Open WebUI does not say so
    # -- knowledge bases still appear, retrieval just returns nothing. Re-embed
    # with POST /api/v1/knowledge/reindex (admin). Note that this covers
    # knowledge bases only: files attached directly to a single chat are not
    # re-indexed, so anything worth keeping belongs in a knowledge base.
    - name: qdrant.service
      enabled: true
      contents: |
        [Unit]
        Description=Qdrant vector database
        After=docker.service
        Requires=docker.service
        [Service]
        TimeoutStartSec=0
        # Both containers sit on a private docker network and Qdrant publishes
        # no ports: it ships with no authentication, and nothing outside this
        # host needs to reach it. Creation is idempotent, hence the "-".
        ExecStartPre=-/usr/bin/docker network create openwebui
        ExecStartPre=/usr/bin/docker pull qdrant/qdrant:v1.19.1
        ExecStartPre=-/usr/bin/docker rm --force qdrant
        # Vectors live on the VM disk, so a taint destroys them. That is
        # deliberate -- they are derived data, rebuildable from the source
        # files in Garage (see the reindex note above). The bind mount is what
        # keeps them across container restarts and image pulls, which the
        # docker rm above would otherwise wipe every time.
        ExecStart=/usr/bin/docker run --name qdrant \
          --network openwebui \
          -v /var/lib/qdrant:/qdrant/storage \
          qdrant/qdrant:v1.19.1
        ExecStop=/usr/bin/docker stop qdrant
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target
    # Open Terminal: the sandbox Open WebUI's models get a shell in, for
    # running code, installing packages and working on files.
    #
    # Anything a model does lands inside this container, not on the host: no
    # docker socket, no host paths, no published port. It is reachable only
    # over the private "openwebui" network, so the API key below is defence in
    # depth rather than the only thing standing in the way.
    #
    # Note that it does have outbound network access -- it needs it to install
    # packages -- so a model can reach the rest of the LAN from in here. That
    # is the part worth thinking about before pointing an untrusted model at
    # it; docker's --internal would close it off, but would also break package
    # installation, which is most of the point.
    - name: open-terminal.service
      enabled: true
      contents: |
        [Unit]
        Description=Open Terminal sandbox
        After=docker.service
        Requires=docker.service
        [Service]
        TimeoutStartSec=0
        ExecStartPre=-/usr/bin/docker network create openwebui
        ExecStartPre=/usr/bin/docker pull ghcr.io/open-webui/open-terminal:0.14.0
        ExecStartPre=-/usr/bin/docker rm --force open-terminal
        # A named volume, not a bind mount: /home/user belongs to the image's
        # unprivileged user, and a named volume inherits that ownership from
        # the image where a root-owned bind mount would not. It lives under
        # /var/lib/docker on the VM disk, so a taint discards the workspace --
        # the right lifetime for scratch space.
        ExecStart=/usr/bin/docker run --name open-terminal \
          --network openwebui \
          -v open-terminal:/home/user \
          -e OPEN_TERMINAL_API_KEY=${openwebui_terminal_api_key} \
          ghcr.io/open-webui/open-terminal:0.14.0
        ExecStop=/usr/bin/docker stop open-terminal
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target
    - name: open-webui.service
      enabled: true
      contents: |
        [Unit]
        Description=Open WebUI
        # After= orders the start only, it does not wait for Qdrant to be
        # serving. Restart=always covers the gap if Qdrant is still coming up.
        After=docker.service litellm.service qdrant.service open-terminal.service
        Requires=docker.service
        [Service]
        TimeoutStartSec=0
        # Large image, so it is pulled up front: a slow first pull is then not
        # mistaken for a hung unit.
        ExecStartPre=/usr/bin/docker pull ghcr.io/open-webui/open-webui:v0.11.4
        ExecStartPre=-/usr/bin/docker network create openwebui
        ExecStartPre=-/usr/bin/docker rm --force open-webui
        # No volume is mounted. Every piece of durable state lives off-box --
        # Postgres for app data, Garage for uploads -- so /app/backend/data
        # holds nothing worth keeping and the container can be recreated at
        # will. Qdrant is the one exception, and it is expendable by design.
        #
        # gitea-mcp already owns host :8080, so the UI is published on :3000.
        ExecStart=/usr/bin/docker run --name open-webui \
          --network openwebui \
          -e DATABASE_URL=${openwebui_database_url} \
          -e VECTOR_DB=qdrant \
          -e QDRANT_URI=http://qdrant:6333 \
          -e QDRANT_ON_DISK=true \
          -e STORAGE_PROVIDER=s3 \
          -e S3_ENDPOINT_URL=http://s3.lan.oswaldtheotter.net:30188 \
          -e S3_REGION_NAME=garage \
          -e S3_BUCKET_NAME=openwebui \
          -e S3_ACCESS_KEY_ID=${openwebui_s3_access_key} \
          -e S3_SECRET_ACCESS_KEY=${openwebui_s3_secret_key} \
          -e WEBUI_SECRET_KEY=${openwebui_secret_key} \
          -e ENABLE_OLLAMA_API=false \
          -e OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \
          -e OPENAI_API_KEY=${litellm_master_key} \
          -e RAG_EMBEDDING_ENGINE=openai \
          -e RAG_OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \
          -e RAG_OPENAI_API_KEY=${litellm_master_key} \
          -e RAG_EMBEDDING_MODEL=text-embedding-3-small \
          -e OAUTH_CLIENT_ID=${openwebui_oauth_client_id} \
          -e OAUTH_CLIENT_SECRET=${openwebui_oauth_client_secret} \
          -e OPENID_PROVIDER_URL=${openwebui_oidc_provider_url} \
          -e OAUTH_PROVIDER_NAME=AtlasHorizon \
          -e ENABLE_OAUTH_SIGNUP=true \
          -e OAUTH_UPDATE_PICTURE_ON_LOGIN=true \
          -e ENABLE_OAUTH_GROUP_MANAGEMENT=true \
          -e OAUTH_PICTURE_CLAIM=avatar \
          -e "OAUTH_SCOPES=openid email profile groups avatar" \
          -e OAUTH_MERGE_ACCOUNTS_BY_EMAIL=true \
          -e WEBUI_URL=https://llm.atlashorizon.net \
          -e 'TERMINAL_SERVER_CONNECTIONS=[{"id":"local","name":"Open Terminal","url":"http://open-terminal:8000","key":"${openwebui_terminal_api_key}","auth_type":"bearer","enabled":true}]' \
          -p 3000:8080 \
          ghcr.io/open-webui/open-webui:v0.11.4
        ExecStop=/usr/bin/docker stop open-webui
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target
    # OpenCode headless server for n8n (and anything else on the LAN).
    #
    # Prerequisites:
    #   - OPENCODE_SERVER_PASSWORD in Infisical (see secrets.tmpl)
    #   - model keys in /etc/opencode/opencode.json matching LiteLLM
    #     model_name values (Admin UI or GET /v1/models). Models live in
    #     the LiteLLM database, so this file cannot discover them.
    #
    # Listens on :4096 with HTTP Basic Auth (user "opencode"). n8n on
    # oswald-runner reaches it at http://192.168.7.61:4096 -- install the
    # n8n-nodes-opencode-ai community package, or use HTTP Request nodes
    # against the OpenAPI at /doc. LLM calls go through LiteLLM on :4000
    # using the same host-IP pattern as Open WebUI.
    - name: opencode.service
      enabled: true
      contents: |
        [Unit]
        Description=OpenCode server
        After=docker.service litellm.service
        Requires=docker.service
        [Service]
        TimeoutStartSec=0
        ExecStartPre=/usr/bin/docker pull ghcr.io/anomalyco/opencode:1.18.32
        ExecStartPre=-/usr/bin/docker rm --force opencode
        # /var/lib/opencode is the project workspace the agent edits. It
        # survives container restarts; a VM taint wipes it (scratch space).
        ExecStart=/usr/bin/docker run --name opencode \
          -e OPENCODE_SERVER_USERNAME=opencode \
          -e OPENCODE_SERVER_PASSWORD=${opencode_server_password} \
          -e OPENCODE_CONFIG=/etc/opencode/opencode.json \
          -v /etc/opencode/opencode.json:/etc/opencode/opencode.json:ro \
          -v /var/lib/opencode:/workspace \
          -w /workspace \
          -p 4096:4096 \
          ghcr.io/anomalyco/opencode:1.18.32 \
          serve --hostname 0.0.0.0 --port 4096
        ExecStop=/usr/bin/docker stop opencode
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target
    # Rebuilds the vector store after a taint. Runs on every boot, but only
    # acts when Qdrant holds no Open WebUI collections -- that is, when the
    # local volume went with the VM.
    #
    # The admin API key it authenticates with lives in Postgres, so it
    # survives the rebuild that wipes Qdrant. That is what makes this work.
    - name: openwebui-reindex.service
      enabled: true
      contents: |
        [Unit]
        Description=Rebuild Open WebUI vectors when Qdrant is empty
        After=open-webui.service qdrant.service
        Requires=open-webui.service qdrant.service
        [Service]
        Type=oneshot
        TimeoutStartSec=30min
        ExecStart=/opt/bin/openwebui-reindex.sh
        [Install]
        WantedBy=multi-user.target
    - name: dozzle-agent.service
      enabled: true
      contents: |
        [Unit]
        Description=Dozzle Agent
        After=docker.service
        Requires=docker.service
        [Service]
        TimeoutStartSec=0
        ExecStartPre=-/usr/bin/docker rm --force dozzle-agent
        ExecStart=/usr/bin/docker run --name dozzle-agent \
          -v /var/run/docker.sock:/var/run/docker.sock \
          -p 7007:7007 \
          amir20/dozzle:v11.1.1 agent
        ExecStop=/usr/bin/docker stop dozzle-agent
        Restart=always
        RestartSec=5s
        [Install]
        WantedBy=multi-user.target

storage:
  directories:
    # Qdrant's vector store. On the VM disk on purpose: a taint is meant to
    # discard it. The qdrant image runs as root (ARG USER_ID=0), so a
    # root-owned 0700 directory is what it expects.
    - path: /var/lib/qdrant
      mode: 0700
    # OpenCode project workspace. Scratch space: survives container
    # restarts, not a VM taint.
    - path: /var/lib/opencode
      mode: 0755
    - path: /etc/opencode
      mode: 0755
  files:
    # LiteLLM proxy settings. Models stay in the database
    # (STORE_MODEL_IN_DB), so this file carries general_settings only --
    # those have no environment equivalent and are readable only from a
    # config file.
    #
    # Requests reach :4000 through a Cloudflare tunnel whose connector runs
    # on 192.168.0.0/24, so every request arrives from that connector's
    # address. Without use_x_forwarded_for LiteLLM does MCP access control
    # against that peer, which sits inside the default internal ranges, and
    # so treats every caller on the public internet as internal -- exposing
    # MCP servers marked "available_on_public_internet: false".
    #
    # mcp_trusted_proxy_ranges is what keeps that safe: X-Forwarded-For is
    # honoured only when the direct peer is the connector, so a crafted
    # header from anywhere else cannot forge a client IP. Widen it if the
    # tunnel ever moves off that subnet, or the header is ignored again.
    # Companion to openwebui-reindex.service above.
    #
    # Note on quoting: templatefile() substitutes on raw text, so shell
    # variables are written as $VAR without braces. A braced shell variable
    # anywhere in this script would be read as a template reference and fail
    # the plan; only the interpolation below is meant to be substituted.
    - path: /opt/bin/openwebui-reindex.sh
      mode: 0700
      contents:
        inline: |
          #!/usr/bin/bash
          set -euo pipefail
          OWUI=http://127.0.0.1:3000
          OWUI_KEY='${openwebui_admin_api_key}'

          # Qdrant publishes no ports -- it is reachable only on the private
          # "openwebui" docker network -- so it is probed from inside the
          # open-webui container, which is on that network and ships curl.
          qdrant_curl() {
            docker exec open-webui curl -fsS "$@"
          }

          # Wait up to 10 minutes for both services to answer.
          ready=0
          for i in $(seq 1 120); do
            if curl -fsS "$OWUI/health" >/dev/null 2>&1 \
               && qdrant_curl http://qdrant:6333/readyz >/dev/null 2>&1; then
              ready=1; break
            fi
            sleep 5
          done
          [ "$ready" = 1 ] || { echo "services never became ready"; exit 1; }

          # Collections are named "<QDRANT_COLLECTION_PREFIX>_<id>", and the
          # prefix defaults to open-webui. Any hit means the volume survived.
          if qdrant_curl http://qdrant:6333/collections \
               | grep -q '"name":"open-webui'; then
            echo "Qdrant already populated; skipping reindex"
            exit 0
          fi

          echo "Qdrant empty; reindexing knowledge bases"
          curl -fsS -X POST -H "Authorization: Bearer $OWUI_KEY" \
            "$OWUI/api/v1/knowledge/reindex"
    - path: /etc/litellm/config.yaml
      mode: 0644
      contents:
        inline: |
          general_settings:
            use_x_forwarded_for: true
            mcp_trusted_proxy_ranges:
              - 192.168.0.0/24
    # OpenCode provider config. LiteLLM is an OpenAI-compatible proxy, so
    # we use @ai-sdk/openai-compatible rather than a first-party provider.
    #
    # Model keys under "models" must match LiteLLM model_name values exactly
    # (they are stored in the LiteLLM DB, not in litellm/config.yaml). Edit
    # this list to match what /v1/models returns, then taint the VM -- or
    # PATCH http://192.168.7.61:4096/config at runtime. The default model
    # below is the first key; change "model" when you change the set.
    #
    # apiKey is templated from Infisical (LITELLM_MASTER_KEY). Ignition
    # snippets are plaintext on the node -- same caveat as the other units.
    - path: /etc/opencode/opencode.json
      mode: 0600
      contents:
        inline: |
          {
            "$schema": "https://opencode.ai/config.json",
            "model": "litellm/claude-sonnet-4",
            "provider": {
              "litellm": {
                "npm": "@ai-sdk/openai-compatible",
                "name": "LiteLLM",
                "options": {
                  "baseURL": "http://192.168.7.61:4000/v1",
                  "apiKey": "${litellm_master_key}"
                },
                "models": {
                  "claude-sonnet-4": {
                    "name": "Claude Sonnet 4"
                  }
                }
              }
            }
          }