variant: flatcar version: 1.0.0 systemd: units: - name: gitea-mcp.service enabled: true contents: | [Unit] Description=Gitea MCP Server After=docker.service versions-fetch.service Wants=versions-fetch.service Requires=docker.service [Service] # Image tag comes from versions.env; see butane/versions-sync.yaml. EnvironmentFile=/etc/versions.env TimeoutStartSec=0 ExecStartPre=-/usr/bin/docker rm --force gitea-mcp # The image ships a broken HEALTHCHECK: it is CMD-SHELL form but the # image is distroless, so every probe fails on a missing /bin/sh and # the container is reported unhealthy while serving fine. Exec-form # health tests cannot be set from docker run, so the check is dropped; # "/app/gitea-mcp -healthcheck" still works via docker exec. ExecStart=/usr/bin/docker run --name gitea-mcp --no-healthcheck \ -e GITEA_HOST=https://git.atlashorizon.net \ -p 8080:8080 \ $${GITEA_MCP_IMAGE} \ /app/gitea-mcp -t http --port 8080 ExecStop=/usr/bin/docker stop gitea-mcp Restart=always RestartSec=5s [Install] WantedBy=multi-user.target - name: litellm.service enabled: true contents: | [Unit] Description=LiteLLM proxy After=docker.service versions-fetch.service Wants=versions-fetch.service Requires=docker.service [Service] # Image tag comes from versions.env; see butane/versions-sync.yaml. EnvironmentFile=/etc/versions.env TimeoutStartSec=0 ExecStartPre=-/usr/bin/docker rm --force litellm ExecStart=/usr/bin/docker run --name litellm \ -e DATABASE_URL=${litellm_database_url} \ -e STORE_MODEL_IN_DB=True \ -e LITELLM_MASTER_KEY=${litellm_master_key} \ -e LITELLM_SALT_KEY=${litellm_salt_key} \ -e LITELLM_LOG=ERROR \ -e JSON_LOGS=True \ -v /etc/litellm/config.yaml:/app/config.yaml:ro \ -p 4000:4000 \ $${LITELLM_IMAGE} \ --config /app/config.yaml --port 4000 ExecStop=/usr/bin/docker stop litellm Restart=always RestartSec=5s [Install] WantedBy=multi-user.target # Open WebUI, with Qdrant alongside it for document embeddings. # # Prerequisites, all of which must exist before the VM is rebuilt: # - Postgres database (plain; no pgvector extension needed -- embeddings # go to Qdrant, and Postgres only holds users, chats and settings) # - openwebui bucket in Garage, plus a key with r/w on it # - an embedding model in LiteLLM matching RAG_EMBEDDING_MODEL below # - an OIDC client at the provider, with the callback URI registered as # https://llm.atlashorizon.net/oauth/oidc/callback # - llm.atlashorizon.net terminating TLS and proxying to this host's # :3000 (not managed here -- see WEBUI_URL below) # - the OPENWEBUI_* secrets in Infisical (see secrets.tmpl) # # After a taint: the vector store is empty and Open WebUI does not say so # -- knowledge bases still appear, retrieval just returns nothing. Re-embed # with POST /api/v1/knowledge/reindex (admin). Note that this covers # knowledge bases only: files attached directly to a single chat are not # re-indexed, so anything worth keeping belongs in a knowledge base. - name: qdrant.service enabled: true contents: | [Unit] Description=Qdrant vector database After=docker.service versions-fetch.service Wants=versions-fetch.service Requires=docker.service [Service] # Image tag comes from versions.env; see butane/versions-sync.yaml. EnvironmentFile=/etc/versions.env TimeoutStartSec=0 # Both containers sit on a private docker network and Qdrant publishes # no ports: it ships with no authentication, and nothing outside this # host needs to reach it. Creation is idempotent, hence the "-". ExecStartPre=-/usr/bin/docker network create openwebui ExecStartPre=/usr/bin/docker pull $${QDRANT_IMAGE} ExecStartPre=-/usr/bin/docker rm --force qdrant # Vectors live on the VM disk, so a taint destroys them. That is # deliberate -- they are derived data, rebuildable from the source # files in Garage (see the reindex note above). The bind mount is what # keeps them across container restarts and image pulls, which the # docker rm above would otherwise wipe every time. ExecStart=/usr/bin/docker run --name qdrant \ --network openwebui \ -v /var/lib/qdrant:/qdrant/storage \ $${QDRANT_IMAGE} ExecStop=/usr/bin/docker stop qdrant Restart=always RestartSec=5s [Install] WantedBy=multi-user.target # Open Terminal: the sandbox Open WebUI's models get a shell in, for # running code, installing packages and working on files. # # Anything a model does lands inside this container, not on the host: no # docker socket, no host paths, no published port. It is reachable only # over the private "openwebui" network, so the API key below is defence in # depth rather than the only thing standing in the way. # # Note that it does have outbound network access -- it needs it to install # packages -- so a model can reach the rest of the LAN from in here. That # is the part worth thinking about before pointing an untrusted model at # it; docker's --internal would close it off, but would also break package # installation, which is most of the point. - name: open-terminal.service enabled: true contents: | [Unit] Description=Open Terminal sandbox After=docker.service versions-fetch.service Wants=versions-fetch.service Requires=docker.service [Service] # Image tag comes from versions.env; see butane/versions-sync.yaml. EnvironmentFile=/etc/versions.env TimeoutStartSec=0 ExecStartPre=-/usr/bin/docker network create openwebui ExecStartPre=/usr/bin/docker pull $${OPEN_TERMINAL_IMAGE} ExecStartPre=-/usr/bin/docker rm --force open-terminal # A named volume, not a bind mount: /home/user belongs to the image's # unprivileged user, and a named volume inherits that ownership from # the image where a root-owned bind mount would not. It lives under # /var/lib/docker on the VM disk, so a taint discards the workspace -- # the right lifetime for scratch space. ExecStart=/usr/bin/docker run --name open-terminal \ --network openwebui \ -v open-terminal:/home/user \ -e OPEN_TERMINAL_API_KEY=${openwebui_terminal_api_key} \ $${OPEN_TERMINAL_IMAGE} ExecStop=/usr/bin/docker stop open-terminal Restart=always RestartSec=5s [Install] WantedBy=multi-user.target - name: open-webui.service enabled: true contents: | [Unit] Description=Open WebUI # After= orders the start only, it does not wait for Qdrant to be # serving. Restart=always covers the gap if Qdrant is still coming up. After=docker.service litellm.service qdrant.service open-terminal.service versions-fetch.service Wants=versions-fetch.service Requires=docker.service [Service] # Image tag comes from versions.env; see butane/versions-sync.yaml. EnvironmentFile=/etc/versions.env TimeoutStartSec=0 # Large image, so it is pulled up front: a slow first pull is then not # mistaken for a hung unit. ExecStartPre=/usr/bin/docker pull $${OPEN_WEBUI_IMAGE} ExecStartPre=-/usr/bin/docker network create openwebui ExecStartPre=-/usr/bin/docker rm --force open-webui # No volume is mounted. Every piece of durable state lives off-box -- # Postgres for app data, Garage for uploads -- so /app/backend/data # holds nothing worth keeping and the container can be recreated at # will. Qdrant is the one exception, and it is expendable by design. # # gitea-mcp already owns host :8080, so the UI is published on :3000. ExecStart=/usr/bin/docker run --name open-webui \ --network openwebui \ -e DATABASE_URL=${openwebui_database_url} \ -e VECTOR_DB=qdrant \ -e QDRANT_URI=http://qdrant:6333 \ -e QDRANT_ON_DISK=true \ -e STORAGE_PROVIDER=s3 \ -e S3_ENDPOINT_URL=http://s3.lan.oswaldtheotter.net:30188 \ -e S3_REGION_NAME=garage \ -e S3_BUCKET_NAME=openwebui \ -e S3_ACCESS_KEY_ID=${openwebui_s3_access_key} \ -e S3_SECRET_ACCESS_KEY=${openwebui_s3_secret_key} \ -e WEBUI_SECRET_KEY=${openwebui_secret_key} \ -e ENABLE_OLLAMA_API=false \ -e OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \ -e OPENAI_API_KEY=${litellm_master_key} \ -e RAG_EMBEDDING_ENGINE=openai \ -e RAG_OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \ -e RAG_OPENAI_API_KEY=${litellm_master_key} \ -e RAG_EMBEDDING_MODEL=text-embedding-3-small \ -e OAUTH_CLIENT_ID=${openwebui_oauth_client_id} \ -e OAUTH_CLIENT_SECRET=${openwebui_oauth_client_secret} \ -e OPENID_PROVIDER_URL=${openwebui_oidc_provider_url} \ -e OAUTH_PROVIDER_NAME=AtlasHorizon \ -e ENABLE_OAUTH_SIGNUP=true \ -e OAUTH_UPDATE_PICTURE_ON_LOGIN=true \ -e ENABLE_OAUTH_GROUP_MANAGEMENT=true \ -e OAUTH_PICTURE_CLAIM=avatar \ -e "OAUTH_SCOPES=openid email profile groups avatar" \ -e OAUTH_MERGE_ACCOUNTS_BY_EMAIL=true \ -e CORS_ALLOW_ORIGIN=https://llm.atlashorizon.net \ -e WEBUI_URL=https://llm.atlashorizon.net \ -e 'TERMINAL_SERVER_CONNECTIONS=[{"id":"local","name":"Open Terminal","url":"http://open-terminal:8000","key":"${openwebui_terminal_api_key}","auth_type":"bearer","enabled":true}]' \ -p 3000:8080 \ $${OPEN_WEBUI_IMAGE} ExecStop=/usr/bin/docker stop open-webui Restart=always RestartSec=5s [Install] WantedBy=multi-user.target # OpenCode, run headless as a coding agent that n8n drives over HTTP. # # Prerequisites, all of which must exist before the VM is rebuilt: # - OPENCODE_SERVER_PASSWORD in Infisical (see secrets.tmpl). Every # Butane file is rendered on every plan, so until the secret exists # no plan succeeds -- not just this VM's # - each model listed in /etc/opencode/opencode.json (below) defined # in LiteLLM under the same model_name # # Unlike Open Terminal, this publishes a port: n8n runs on oswald-runner, # so the private "openwebui" network is out of its reach. That leaves HTTP # Basic Auth (user "opencode") as the only thing between the LAN and an # agent with a shell, and over plain HTTP. The agent runs as root inside # the container, with the same outbound network access as Open Terminal. # # From n8n: http://192.168.7.61:4096, with the n8n-nodes-opencode-ai # community node or HTTP Request nodes (the OpenAPI spec is at /doc). - name: opencode.service enabled: true contents: | [Unit] Description=OpenCode server After=docker.service litellm.service versions-fetch.service Wants=versions-fetch.service Requires=docker.service [Service] # Image tag comes from versions.env; see butane/versions-sync.yaml. EnvironmentFile=/etc/versions.env TimeoutStartSec=0 ExecStartPre=/usr/bin/docker pull $${OPENCODE_IMAGE} ExecStartPre=-/usr/bin/docker rm --force opencode # Both directories are on the VM disk, so they survive container # restarts but not a taint. workspace is what the agent edits; data # holds its sessions, which n8n refers to by id across executions. # # The LiteLLM key is passed in the environment and read by the config # file as {env:LITELLM_API_KEY}, which keeps the file free of secrets. ExecStart=/usr/bin/docker run --name opencode \ -e OPENCODE_SERVER_USERNAME=opencode \ -e OPENCODE_SERVER_PASSWORD=${opencode_server_password} \ -e LITELLM_API_KEY=${litellm_master_key} \ -e OPENCODE_CONFIG=/etc/opencode/opencode.json \ -v /etc/opencode/opencode.json:/etc/opencode/opencode.json:ro \ -v /var/lib/opencode/workspace:/workspace \ -v /var/lib/opencode/data:/root/.local/share/opencode \ -w /workspace \ -p 4096:4096 \ $${OPENCODE_IMAGE} \ serve --hostname 0.0.0.0 --port 4096 ExecStop=/usr/bin/docker stop opencode Restart=always RestartSec=5s [Install] WantedBy=multi-user.target # Rebuilds the vector store after a taint. Runs on every boot, but only # acts when Qdrant holds no Open WebUI collections -- that is, when the # local volume went with the VM. # # The admin API key it authenticates with lives in Postgres, so it # survives the rebuild that wipes Qdrant. That is what makes this work. - name: openwebui-reindex.service enabled: true contents: | [Unit] Description=Rebuild Open WebUI vectors when Qdrant is empty After=open-webui.service qdrant.service Requires=open-webui.service qdrant.service [Service] Type=oneshot TimeoutStartSec=30min ExecStart=/opt/bin/openwebui-reindex.sh [Install] WantedBy=multi-user.target - name: dozzle-agent.service enabled: true contents: | [Unit] Description=Dozzle Agent After=docker.service versions-fetch.service Wants=versions-fetch.service Requires=docker.service [Service] # Image tag comes from versions.env; see butane/versions-sync.yaml. EnvironmentFile=/etc/versions.env TimeoutStartSec=0 ExecStartPre=-/usr/bin/docker rm --force dozzle-agent ExecStart=/usr/bin/docker run --name dozzle-agent \ -v /var/run/docker.sock:/var/run/docker.sock \ -p 7007:7007 \ $${DOZZLE_IMAGE} agent ExecStop=/usr/bin/docker stop dozzle-agent Restart=always RestartSec=5s [Install] WantedBy=multi-user.target storage: directories: # Qdrant's vector store. On the VM disk on purpose: a taint is meant to # discard it. The qdrant image runs as root (ARG USER_ID=0), so a # root-owned 0700 directory is what it expects. - path: /var/lib/qdrant mode: 0700 # OpenCode's workspace and session store; see opencode.service. The # image runs as root. - path: /var/lib/opencode/workspace mode: 0755 - path: /var/lib/opencode/data mode: 0700 files: # Companion to openwebui-reindex.service above. # # Note on quoting: templatefile() substitutes on raw text, so shell # variables are written as $VAR without braces. A braced shell variable # anywhere in this script would be read as a template reference and fail # the plan; only the interpolation below is meant to be substituted. - path: /opt/bin/openwebui-reindex.sh mode: 0700 contents: inline: | #!/usr/bin/bash set -euo pipefail OWUI=http://127.0.0.1:3000 OWUI_KEY='${openwebui_admin_api_key}' # Qdrant publishes no ports -- it is reachable only on the private # "openwebui" docker network -- so it is probed from inside the # open-webui container, which is on that network and ships curl. qdrant_curl() { docker exec open-webui curl -fsS "$@" } # Wait up to 10 minutes for both services to answer. ready=0 for i in $(seq 1 120); do if curl -fsS "$OWUI/health" >/dev/null 2>&1 \ && qdrant_curl http://qdrant:6333/readyz >/dev/null 2>&1; then ready=1; break fi sleep 5 done [ "$ready" = 1 ] || { echo "services never became ready"; exit 1; } # Collections are named "_", and the # prefix defaults to open-webui. Any hit means the volume survived. if qdrant_curl http://qdrant:6333/collections \ | grep -q '"name":"open-webui'; then echo "Qdrant already populated; skipping reindex" exit 0 fi echo "Qdrant empty; reindexing knowledge bases" curl -fsS -X POST -H "Authorization: Bearer $OWUI_KEY" \ "$OWUI/api/v1/knowledge/reindex" # LiteLLM proxy settings. Models stay in the database # (STORE_MODEL_IN_DB), so this file carries general_settings only -- # those have no environment equivalent and are readable only from a # config file. # # Requests reach :4000 through a Cloudflare tunnel whose connector runs # on 192.168.0.0/24, so every request arrives from that connector's # address. Without use_x_forwarded_for LiteLLM does MCP access control # against that peer, which sits inside the default internal ranges, and # so treats every caller on the public internet as internal -- exposing # MCP servers marked "available_on_public_internet: false". # # mcp_trusted_proxy_ranges is what keeps that safe: X-Forwarded-For is # honoured only when the direct peer is the connector, so a crafted # header from anywhere else cannot forge a client IP. Widen it if the # tunnel ever moves off that subnet, or the header is ignored again. - path: /etc/litellm/config.yaml mode: 0644 contents: inline: | general_settings: use_x_forwarded_for: true mcp_trusted_proxy_ranges: - 192.168.0.0/24 # OpenCode's provider config. LiteLLM speaks the OpenAI API, so it is # wired in through @ai-sdk/openai-compatible. # # Model keys must match LiteLLM model_name values exactly. Those live in # the LiteLLM database, not in a file here, so check GET :4000/v1/models # when changing this list. "model" is the default for new sessions. - path: /etc/opencode/opencode.json mode: 0644 contents: inline: | { "$schema": "https://opencode.ai/config.json", "model": "litellm/claude-sonnet-5", "provider": { "litellm": { "npm": "@ai-sdk/openai-compatible", "name": "LiteLLM", "options": { "baseURL": "http://192.168.7.61:4000/v1", "apiKey": "{env:LITELLM_API_KEY}" }, "models": { "claude-sonnet-5": { "name": "Claude Sonnet 5" }, "qwen3:4b-instruct": { "name": "Qwen3 4B Instruct" } } } } }