From 4910bc3fade7da84b3ec0871c213adf965380009 Mon Sep 17 00:00:00 2001 From: Chewbaccalakis Date: Wed, 23 Sep 2026 06:45:19 +0000 Subject: [PATCH] fixed health check, added disabled openwebui --- butane/ai.yaml.tftpl | 90 +++++++++++++++++++++++++++++++- butane/oswl_connector.yaml.tftpl | 13 ++++- flatcar-todo.md | 68 ++++++++++++++---------- 3 files changed, 140 insertions(+), 31 deletions(-) diff --git a/butane/ai.yaml.tftpl b/butane/ai.yaml.tftpl index 7d0161a..c9f5410 100644 --- a/butane/ai.yaml.tftpl +++ b/butane/ai.yaml.tftpl @@ -13,7 +13,12 @@ systemd: [Service] TimeoutStartSec=0 ExecStartPre=-/usr/bin/docker rm --force gitea-mcp - ExecStart=/usr/bin/docker run --name gitea-mcp \ + # The image ships a broken HEALTHCHECK: it is CMD-SHELL form but the + # image is distroless, so every probe fails on a missing /bin/sh and + # the container is reported unhealthy while serving fine. Exec-form + # health tests cannot be set from docker run, so the check is dropped; + # "/app/gitea-mcp -healthcheck" still works via docker exec. + ExecStart=/usr/bin/docker run --name gitea-mcp --no-healthcheck \ -e GITEA_HOST=https://git.atlashorizon.net \ -p 8080:8080 \ docker.gitea.com/gitea-mcp-server \ @@ -45,6 +50,89 @@ systemd: RestartSec=5s [Install] WantedBy=multi-user.target + # ── Open WebUI (disabled) ────────────────────────────────────────────── + # Commented out until it is ready to launch. To re-enable: + # 1. strip the leading "# " from every line of the block below (this + # restores the original indentation exactly) + # 2. un-double the dollar signs in each interpolation: $$ back to $ + # 3. re-add the seven OPENWEBUI_* keys to secrets.tmpl (listed below; + # they are removed while this is off so the generated JSON, which + # data.external parses, stays valid) + # + # OPENWEBUI_DATABASE_URL -> openwebui_database_url + # OPENWEBUI_SECRET_KEY -> openwebui_secret_key + # OPENWEBUI_S3_ACCESS_KEY -> openwebui_s3_access_key + # OPENWEBUI_S3_SECRET_KEY -> openwebui_s3_secret_key + # OPENWEBUI_OAUTH_CLIENT_ID -> openwebui_oauth_client_id + # OPENWEBUI_OAUTH_CLIENT_SECRET -> openwebui_oauth_client_secret + # OPENWEBUI_OIDC_PROVIDER_URL -> openwebui_oidc_provider_url + # + # The doubling in step 2 is not cosmetic: templatefile() substitutes on + # raw text before anything parses it as YAML, so an interpolation left + # intact here would still be resolved -- and would still fail the plan on + # a missing key -- even though it sits inside a comment. +# # Open WebUI. +# # +# # Prerequisites, all of which must exist before the VM is rebuilt: +# # - Postgres database, with "CREATE EXTENSION vector;" run inside it +# # - openwebui bucket in Garage, plus a key with r/w on it +# # - an embedding model in LiteLLM matching RAG_EMBEDDING_MODEL below +# # - an OIDC client at the provider, with the callback URI registered as +# # https://llm.atlashorizon.net/oauth/oidc/callback +# # - llm.atlashorizon.net terminating TLS and proxying to this host's +# # :3000 (not managed here -- see WEBUI_URL below) +# # - the OPENWEBUI_* secrets in Infisical (see secrets.tmpl) +# - name: open-webui.service +# enabled: true +# contents: | +# [Unit] +# Description=Open WebUI +# After=docker.service litellm.service +# Requires=docker.service +# [Service] +# TimeoutStartSec=0 +# # Large image, so it is pulled up front: a slow first pull is then not +# # mistaken for a hung unit. +# ExecStartPre=/usr/bin/docker pull ghcr.io/open-webui/open-webui:v0.11.4 +# ExecStartPre=-/usr/bin/docker rm --force open-webui +# # No volume is mounted. Every piece of durable state lives off-box -- +# # Postgres for app data, pgvector for embeddings, Garage for uploads -- +# # so /app/backend/data holds nothing worth keeping and the container +# # (and the whole VM) can be rebuilt at will. What stays local is cache +# # only, re-fetched on the next start. +# # +# # gitea-mcp already owns host :8080, so the UI is published on :3000. +# ExecStart=/usr/bin/docker run --name open-webui \ +# -e DATABASE_URL=$${openwebui_database_url} \ +# -e VECTOR_DB=pgvector \ +# -e PGVECTOR_DB_URL=$${openwebui_database_url} \ +# -e STORAGE_PROVIDER=s3 \ +# -e S3_ENDPOINT_URL=http://s3.lan.oswaldtheotter.net:30188 \ +# -e S3_REGION_NAME=garage \ +# -e S3_BUCKET_NAME=openwebui \ +# -e S3_ACCESS_KEY_ID=$${openwebui_s3_access_key} \ +# -e S3_SECRET_ACCESS_KEY=$${openwebui_s3_secret_key} \ +# -e WEBUI_SECRET_KEY=$${openwebui_secret_key} \ +# -e ENABLE_OLLAMA_API=false \ +# -e OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \ +# -e OPENAI_API_KEY=$${litellm_master_key} \ +# -e RAG_EMBEDDING_ENGINE=openai \ +# -e RAG_OPENAI_API_BASE_URL=http://192.168.7.61:4000/v1 \ +# -e RAG_OPENAI_API_KEY=$${litellm_master_key} \ +# -e RAG_EMBEDDING_MODEL=text-embedding-3-small \ +# -e OAUTH_CLIENT_ID=$${openwebui_oauth_client_id} \ +# -e OAUTH_CLIENT_SECRET=$${openwebui_oauth_client_secret} \ +# -e OPENID_PROVIDER_URL=$${openwebui_oidc_provider_url} \ +# -e OAUTH_PROVIDER_NAME=SSO \ +# -e ENABLE_OAUTH_SIGNUP=true \ +# -e WEBUI_URL=https://llm.atlashorizon.net \ +# -p 3000:8080 \ +# ghcr.io/open-webui/open-webui:v0.11.4 +# ExecStop=/usr/bin/docker stop open-webui +# Restart=always +# RestartSec=5s +# [Install] +# WantedBy=multi-user.target - name: dozzle-agent.service enabled: true contents: | diff --git a/butane/oswl_connector.yaml.tftpl b/butane/oswl_connector.yaml.tftpl index ab9026e..efba39b 100644 --- a/butane/oswl_connector.yaml.tftpl +++ b/butane/oswl_connector.yaml.tftpl @@ -25,8 +25,17 @@ systemd: Wants=network-online.target [Service] TimeoutStartSec=0 - # tun is not loaded by default on a stock Flatcar boot. - ExecStartPre=-/usr/sbin/modprobe tun + # None of these are loaded by default on a stock Flatcar boot, and the + # container cannot load them itself: it has NET_ADMIN (enough to write + # iptables rules) but not SYS_MODULE, and the image ships no + # /lib/modules, so tailscaled's own modprobe fallback fails with + # "the filter table does not exist". Loading them here works because + # the container shares the host's network namespace (--network host), + # so it sees tables created on the host. Without the nat table in + # particular there is no ts-postrouting chain, which means no SNAT: + # the exit node cannot work at all and subnet-routed replies are sent + # back to tailnet addresses the LAN has no route for. + ExecStartPre=-/usr/sbin/modprobe -a tun iptable_filter iptable_nat ip6table_filter ip6table_nat nf_tables xt_mark # Recommended by Tailscale for subnet routers and exit nodes: without # this, UDP throughput through the node is roughly halved. ExecStartPre=-/usr/bin/bash -c 'ethtool -K $(ip route show 0.0.0.0/0 | cut -d" " -f5) rx-udp-gro-forwarding on rx-gro-list off' diff --git a/flatcar-todo.md b/flatcar-todo.md index 79f7a75..684b6b5 100644 --- a/flatcar-todo.md +++ b/flatcar-todo.md @@ -112,33 +112,26 @@ as an exit node. Three things have to happen outside this repo: It must be both ephemeral and reusable -- the VM keeps tailscaled's state in memory, so it re-registers on every boot: + The key also carries the tag that drives route auto-approval (see 2), so + the VM never has to pass `--advertise-tags` itself: + ```bash - headscale preauthkeys create --user --ephemeral --reusable --expiration 8760h + headscale preauthkeys create --user --ephemeral --reusable \ + --tags tag:oswl-connector --expiration 8760h ``` Until that secret exists, `tofu plan` fails while rendering `butane/oswl_connector.yaml.tftpl` (`templatefile`: no value for "headscale_authkey"). -2. **Approve the route and the exit node** once the VM has registered -- - headscale honours neither until enabled: +2. **Route approval is automatic, via the key's tag.** The preauth key applies + `tag:oswl-connector` as a forced tag at registration, and headscale's + `autoApprovers` policy approves the subnet route and the exit node for that + tag. This is what makes the node's ephemerality survivable: it re-registers + on every boot and after every `tofu taint` rebuild, and each time the routes + come back approved without anyone touching headscale. - ```bash - headscale nodes list - headscale nodes approve-routes -i -r 192.168.7.0/24,0.0.0.0/0,::/0 - ``` - - (On headscale older than 0.26 that is `headscale routes list` plus - `headscale routes enable -r `, once per route.) - - Because the node is ephemeral it re-registers on every boot, so this has to - be redone after each reboot and after each `tofu taint` rebuild. Headscale - can do it automatically, but **the policy has to exist before the node - advertises the tag** -- `tailscale up --advertise-tags` fails outright if - the tag is not owned by the registering user, which takes tailscaled down - with it. So: add the policy first, confirm it loads, and only then add - `--advertise-tags=tag:oswl-connector` to `TS_EXTRA_ARGS` in - `butane/oswl_connector.yaml.tftpl` and rebuild the VM. + The policy this depends on: ```json { @@ -154,21 +147,28 @@ as an exit node. Three things have to happen outside this repo: } ``` - `@` is the user the preauth key belongs to (policy v2 spells - usernames with a trailing `@`). Load the policy with `policy.mode: file` + - `policy.path` in headscale's `config.yaml`, or `headscale policy set -f - policy.json` in database mode. + Two things worth knowing if it ever stops approving: - Two further caveats if you go that route: - - - **List the tag in `autoApprovers`, not a user or a group.** Approval by + - **`autoApprovers` must name the tag, not a user or a group.** Approval by username or group silently does not fire for a node whose identity is a - tag ([#2891](https://github.com/juanfont/headscale/issues/2891), open as - of 0.27.1). Naming the tag is both the documented form and the workaround. + forced tag ([#2891](https://github.com/juanfont/headscale/issues/2891), + open as of 0.27.1). - **A tagged node is governed by the tag in ACLs, not by its user.** With headscale's default allow-all policy that changes nothing, but if you later write real ACL rules, `tag:oswl-connector` needs its own grants. + To approve by hand (if the policy is ever removed): + + ```bash + headscale nodes list + headscale nodes approve-routes -i -r 192.168.7.0/24,0.0.0.0/0,::/0 + ``` + + Do not add `--advertise-tags` to `TS_EXTRA_ARGS` to try to set the tag from + the client side: the key already sets it, and `tailscale up` fails outright + if the registering user does not own the tag, which takes tailscaled down + with it. + 3. **Check the LAN's return path.** The VM has one NIC on VLAN 7, so clients reaching 192.168.7.0/24 through the tunnel arrive with a tailnet source address. The container runs with `--network host` and tailscaled installs @@ -176,6 +176,18 @@ as an exit node. Three things have to happen outside this repo: without touching the router; drop that default only if you add a route for the tailnet CIDR on the 192.168.7.0/24 gateway. + That SNAT is only real if the netfilter modules are loaded -- the container + has NET_ADMIN but no SYS_MODULE and no `/lib/modules`, so it cannot load + them itself. The `modprobe -a` line in `tailscaled.service` loads them on + the host, where the container picks them up through `--network host`. If + tailscale ever reports "packet filter not installed" or + "cannot create the ts-input chain", that line is what to check: + + ```bash + lsmod | grep -E 'iptable|ip6table|nf_nat|nf_tables' + docker exec tailscaled iptables -t nat -S | grep ts-postrouting + ``` + Note the same caveat as the Gitea runner: the auth key ends up in plaintext in the rendered Ignition snippet under `/var/lib/vz/snippets` on goose.