Run oswald-runner, ai and oswl-connector as Flatcar VMs

Replaces the Debian VM that hosted the Gitea Actions runner and adds two
new hosts on goose, all provisioned by Ignition instead of cloud-init:

- oswald-runner: Gitea Actions runner, keeping its registration in
  /var/lib/oswald_runner on disk rather than running ephemeral, which was
  leaving stale offline runners behind whenever a job was interrupted.
- ai: LiteLLM proxy and the Gitea MCP server.
- oswl-connector: tailscaled against headscale, advertising 192.168.7.0/24
  and itself as an exit node.

Their registration tokens and keys come from Infisical through the new
butane_vars plumbing, so nothing secret is committed -- but each one does
land in plaintext in the rendered Ignition snippet on the node, which is
the tradeoff for having no cloud-init here.

The provider gains an ssh block on both nodes because Proxmox has no API
for uploading snippets. That, the snippets content type, and the headscale
preauth key all have to be set up by hand first; flatcar-todo.md has the
steps and the reasoning behind the awkward parts.

Co-Authored-By: Claude Opus 5 <[email protected]>

butane/ai.yaml.tftpl

@@ -0,0 +1,47 @@
+variant: flatcar
+version: 1.0.0
+
+systemd:
+  units:
+    - name: gitea-mcp.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Gitea MCP Server
+        After=docker.service
+        Requires=docker.service
+        [Service]
+        TimeoutStartSec=0
+        ExecStartPre=-/usr/bin/docker rm --force gitea-mcp
+        ExecStart=/usr/bin/docker run --name gitea-mcp \
+          -e GITEA_HOST=https://git.atlashorizon.net \
+          -p 8080:8080 \
+          docker.gitea.com/gitea-mcp-server \
+          /app/gitea-mcp -t http --port 8080
+        ExecStop=/usr/bin/docker stop gitea-mcp
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    - name: litellm.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=LiteLLM proxy
+        After=docker.service
+        Requires=docker.service
+        [Service]
+        TimeoutStartSec=0
+        ExecStartPre=-/usr/bin/docker rm --force litellm
+        ExecStart=/usr/bin/docker run --name litellm \
+          -e DATABASE_URL=${litellm_database_url} \
+          -e STORE_MODEL_IN_DB=True \
+          -e LITELLM_MASTER_KEY=${litellm_master_key} \
+          -e LITELLM_SALT_KEY=${litellm_salt_key} \
+          -p 4000:4000 \
+          docker.litellm.ai/berriai/litellm:main-stable
+        ExecStop=/usr/bin/docker stop litellm
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target

butane/example.yaml

@@ -0,0 +1,36 @@
+# Example extra Butane config for a Flatcar VM. Reference it from tfvars with
+# `butane_file = "butane/<name>.yaml"`. It is merged into the base config the
+# module generates (hostname, core + ansible users with the shared SSH key).
+# Applied by Ignition on FIRST BOOT only: to change a running VM, edit this
+# file and taint the VM so it is rebuilt from the image.
+variant: flatcar
+version: 1.0.0
+
+systemd:
+  units:
+    # Example: run a container as a service.
+    - name: hello.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Hello container
+        After=docker.service
+        Requires=docker.service
+        [Service]
+        TimeoutStartSec=0
+        ExecStartPre=-/usr/bin/docker rm --force hello
+        ExecStart=/usr/bin/docker run --name hello --rm -p 8080:80 nginx:alpine
+        ExecStop=/usr/bin/docker stop hello
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+
+storage:
+  files:
+    # Example: a file on disk.
+    - path: /etc/motd
+      mode: 0644
+      contents:
+        inline: |
+          Managed by OpenTofu + Ignition. Config changes require a rebuild.

butane/oswald_runner.yaml.tftpl

@@ -0,0 +1,49 @@
+variant: flatcar
+version: 1.0.0
+
+systemd:
+  units:
+    # Gitea Actions runner. Registers once on first start and keeps its
+    # identity (the .runner file) in /var/lib/oswald_runner on the VM's own
+    # root disk -- not ephemeral, so it does not self-delete after each job
+    # (that caused stale offline entries whenever the container was
+    # interrupted before finishing a job; see flatcar-todo.md). Reusing the
+    # same registration also means the token never needs rotating.
+    #
+    # /var/lib/oswald_runner survives container restarts and VM reboots (it's
+    # on disk, not in the container), but not a VM rebuild (taint): that
+    # recreates the disk from the image, so the runner re-registers fresh
+    # next boot. Old entries from before this change, or from any future
+    # rebuild, still need manual cleanup in Gitea's runner admin page.
+    #
+    # GITEA_RUNNER_REGISTRATION_TOKEN is templated in from Infisical
+    # (GITEA_RUNNER_TOKEN in secrets.tmpl) by main.tf, not stored here. It
+    # still ends up in plaintext in the rendered Ignition snippet on the
+    # node's storage -- see the caveat in flatcar-todo.md.
+    - name: oswald_runner.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Gitea Actions runner
+        After=docker.service
+        Requires=docker.service
+        [Service]
+        TimeoutStartSec=0
+        ExecStartPre=-/usr/bin/docker rm --force oswald_runner
+        ExecStart=/usr/bin/docker run --name oswald_runner \
+          -e GITEA_INSTANCE_URL=https://git.atlashorizon.net \
+          -e GITEA_RUNNER_REGISTRATION_TOKEN=${gitea_runner_token} \
+          -e GITEA_RUNNER_NAME=oswald_runner \
+          -v /var/lib/oswald_runner:/data \
+          -v /var/run/docker.sock:/var/run/docker.sock \
+          docker.io/gitea/runner:3
+        ExecStop=/usr/bin/docker stop oswald_runner
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+
+storage:
+  directories:
+    - path: /var/lib/oswald_runner
+      mode: 0755

butane/oswl_connector.yaml.tftpl

@@ -0,0 +1,60 @@
+variant: flatcar
+version: 1.0.0
+
+# Headscale (Tailscale) node: advertises 192.168.7.0/24 as a subnet route and
+# offers itself as an exit node.
+#
+# Registered as an EPHEMERAL node: tailscaled runs with in-memory state, so the
+# VM re-authenticates on every boot and headscale drops the node shortly after
+# it goes offline. That requires the preauth key to be both --ephemeral and
+# --reusable, otherwise the second boot fails to register.
+#
+# HEADSCALE_AUTHKEY is templated in from Infisical (see secrets.tmpl) by
+# main.tf, not stored here. It still ends up in plaintext in the rendered
+# Ignition snippet on the node's storage -- see the caveat in flatcar-todo.md.
+
+systemd:
+  units:
+    - name: tailscaled.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Tailscale node (headscale subnet router + exit node)
+        After=docker.service network-online.target
+        Requires=docker.service
+        Wants=network-online.target
+        [Service]
+        TimeoutStartSec=0
+        # tun is not loaded by default on a stock Flatcar boot.
+        ExecStartPre=-/usr/sbin/modprobe tun
+        # Recommended by Tailscale for subnet routers and exit nodes: without
+        # this, UDP throughput through the node is roughly halved.
+        ExecStartPre=-/usr/bin/bash -c 'ethtool -K $(ip route show 0.0.0.0/0 | cut -d" " -f5) rx-udp-gro-forwarding on rx-gro-list off'
+        ExecStartPre=-/usr/bin/docker rm --force tailscaled
+        ExecStart=/usr/bin/docker run --name tailscaled \
+          --network host --cap-add NET_ADMIN --cap-add NET_RAW \
+          --device /dev/net/tun \
+          -e TS_AUTHKEY=${headscale_authkey} \
+          -e TS_HOSTNAME=oswl-connector \
+          -e TS_ROUTES=192.168.7.0/24 \
+          -e TS_ACCEPT_DNS=false \
+          -e "TS_EXTRA_ARGS=--login-server=https://hs.atlashorizon.net --advertise-exit-node" \
+          -e TS_USERSPACE=false \
+          docker.io/tailscale/tailscale:stable
+        ExecStop=/usr/bin/docker stop tailscaled
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+
+storage:
+  files:
+    # Forwarding must be on for the subnet route and the exit node to pass
+    # traffic at all; the container shares the host netns (--network host), so
+    # this has to be set on the host, not inside it.
+    - path: /etc/sysctl.d/99-tailscale.conf
+      mode: 0644
+      contents:
+        inline: |
+          net.ipv4.ip_forward = 1
+          net.ipv6.conf.all.forwarding = 1

butane/webserver.yaml

@@ -0,0 +1,75 @@
+# Butane config for a Flatcar VM running the official httpd (Apache) image
+# as a docker container. Applied by Ignition on FIRST BOOT only: to change a
+# running VM, edit this file and taint the VM so it is rebuilt from the image.
+variant: flatcar
+version: 1.0.0
+
+systemd:
+  units:
+    - name: webserver.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=httpd webserver container
+        After=docker.service
+        Requires=docker.service
+        [Service]
+        TimeoutStartSec=0
+        ExecStartPre=-/usr/bin/docker rm --force webserver
+        ExecStart=/usr/bin/docker run --name webserver --rm -p 80:80 httpd:latest
+        ExecStop=/usr/bin/docker stop webserver
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+    # Experimental: Flatcar ships no qemu-guest-agent binary, so this runs one
+    # from an Alpine container against the virtio-serial channel Proxmox
+    # exposes when the VM's `agent.enabled` is true (see qemu_guest_agent in
+    # tfvars). --network host is required so qemu-ga reports the host's real
+    # network interfaces/IP instead of the container's own bridge IP.
+    # --pid host + the /sbin/shutdown bind mount (see /opt/qga-shutdown.sh
+    # below) are required so guest-shutdown/reboot actually reach the host
+    # instead of Alpine's own busybox init inside the container.
+    # --privileged + host /dev is broader access than qemu-ga strictly needs,
+    # but keeps this simple for a first try.
+    - name: qemu-guest-agent.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=QEMU guest agent (containerized)
+        After=docker.service
+        Requires=docker.service
+        [Service]
+        TimeoutStartSec=0
+        ExecStartPre=-/usr/bin/docker rm --force qemu-guest-agent
+        ExecStart=/usr/bin/docker run --name qemu-guest-agent --rm --privileged \
+          --network host --pid host -v /dev:/dev \
+          -v /opt/qga-shutdown.sh:/sbin/shutdown:ro \
+          alpine:latest sh -c \
+          "apk add --no-cache qemu-guest-agent util-linux && exec qemu-ga -m virtio-serial -p /dev/virtio-ports/org.qemu.guest_agent.0"
+        Restart=always
+        RestartSec=5s
+        [Install]
+        WantedBy=multi-user.target
+
+storage:
+  files:
+    # qemu-ga always execs "/sbin/shutdown <flag> +0 <reason>" (flag is -H
+    # halt, -P powerdown, -r reboot) to handle guest-shutdown/guest-reboot.
+    # Inside the container that path is Alpine's own busybox shutdown, which
+    # only affects the container's namespace. Bind-mounted over /sbin/shutdown
+    # in the container (see qemu-guest-agent.service above), this shim uses
+    # nsenter to re-enter the real host's namespaces (PID 1 in the container
+    # is the host's actual systemd, since the container runs with --pid host)
+    # and calls the host's real systemctl instead.
+    - path: /opt/qga-shutdown.sh
+      mode: 0755
+      contents:
+        inline: |
+          #!/bin/sh
+          case "$1" in
+            -H) action=halt ;;
+            -r) action=reboot ;;
+            *)  action=poweroff ;;
+          esac
+          exec nsenter --target 1 --mount --uts --ipc --net -- systemctl "$action"

flatcar-todo.md

@@ -0,0 +1,187 @@
+# Flatcar on Proxmox: what is left for you
+
+The OpenTofu side is done: `nodes.<hv>.flatcar` in `terraform.tfvars` now creates
+Flatcar VMs (see the Flatcar section in `opentofu/README.md` for how it works).
+The items below need access or decisions I do not have. Nothing in this list
+touches the existing LXCs/VMs.
+
+## 1. Enable the Snippets content type on `local` (both nodes)
+
+`local` already has `import` enabled (needed for the image), but not `snippets`
+(needed for the Ignition file). On each node, as root:
+
+```bash
+pvesm set local --content backup,vztmpl,import,iso,snippets
+```
+
+Or in the UI: Datacenter → Storage → local → Edit → Content → tick Snippets.
+
+Proxmox does **not** create the snippets directory when you enable the content
+type -- you have to do it yourself, or the upload fails with `tee:
+/var/lib/vz/snippets/<name>-ignition.json: No such file or directory`:
+
+```bash
+mkdir -p /var/lib/vz/snippets   # confirm the path via `local`'s entry in /etc/pve/storage.cfg
+```
+
+## 2. Let the provider SSH to the nodes as root (both nodes)
+
+Proxmox has no API for uploading snippets, so bpg copies them over SSH. The
+provider is configured (`providers.tf`, `ssh` block) to log in as `root` with
+the ansible key from Infisical (`SSH_PRIVATE_KEY` / `SSH_PUBLIC_KEY`). On each
+node append the public key to root's authorized_keys:
+
+```bash
+# from this machine; the key file is materialized by tofu at ~/.ssh/infra_ansible_key
+ssh-copy-id -i ~/.ssh/infra_ansible_key.pub [email protected]   # pve
+ssh-copy-id -i ~/.ssh/infra_ansible_key.pub [email protected]   # goose
+```
+
+(`~/.ssh/infra_ansible_key.pub` may not exist yet; paste `SSH_PUBLIC_KEY` from
+Infisical into `/root/.ssh/authorized_keys` on the node instead.)
+
+Then verify from here:
+
+```bash
+ssh -i ~/.ssh/infra_ansible_key [email protected] 'pvesm status'
+```
+
+If you would rather not give the provider root, bpg supports a non-root user
+with a narrow sudoers file (`pvesm`, `qm`, `tee /var/lib/vz/snippets/*`). Change
+`username` in the `ssh` blocks accordingly.
+
+## 3. Declare a VM
+
+Uncomment and adapt the `flatcar = { ... }` example under `nodes.goose` in
+`terraform.tfvars`. Pick an unused vmid and IP. Optional per-VM config goes in
+`butane/<name>.yaml`, referenced with `butane_file = "butane/<name>.yaml"` on
+the VM (`butane/example.yaml` is a starting point; keep `variant: flatcar` and
+`version: 1.0.0`). Inline YAML via `butane = <<-EOT ... EOT` also works.
+
+I dry-ran a plan with the example VM declared (nothing applied): it creates
+the image download, the Ignition snippet, the VM and its DNS record, and
+changes nothing else.
+
+Then:
+
+```bash
+infisical run --path /terraform -- tofu plan -out plan
+infisical run --path /terraform -- tofu apply plan
+```
+
+The first apply on a node downloads the 500 MB Flatcar image into `local`
+(`flatcar-stable-4593.2.5.img`), then creates the VM with that image imported
+as its boot disk. Expect the download to take a minute or two.
+
+Log in with `ssh ansible@<ip>` or `ssh core@<ip>` using the ansible key.
+
+## 4. Things to know
+
+- Ignition runs on first boot only. Editing the Butane file updates the snippet on the
+  node but does nothing to a running VM. To apply, rebuild it:
+  `tofu taint 'module.goose.proxmox_virtual_environment_vm.flatcar["<name>"]'`
+  (this destroys the VM's disk; keep data in volumes you can recreate).
+- Ignition snippets are plain text on the node. Do not put secrets in `butane`.
+- Flatcar has no Python, so the existing Ansible plays will not run against it.
+  These hosts land in a `flatcar` group in the generated inventory; only
+  `raw`/`script` tasks work there. Ignition is the intended replacement.
+- No QEMU guest agent: the module sets `agent.enabled = false` for Flatcar VMs,
+  so plans never wait on it. Proxmox will not show the VM's IP in the UI.
+- Bumping `flatcar_image.version` (in `vars.tf` default or tfvars) plus its
+  sha512 from the release `.DIGESTS` file downloads a new image for future VMs
+  only; existing VMs are not touched (`import_from` is ignored after creation).
+  Flatcar itself auto-updates in place via its update engine.
+
+## 5. If the VM does not boot
+
+The Flatcar docs pass `--args "-machine sata=off"` when creating the VM. I left
+it out: that option is root-only in Proxmox and the API token cannot set it.
+If a Flatcar VM hangs at boot, check the serial console (`qm terminal <vmid>`)
+first; if it looks firmware-related, set that arg once by hand as root
+(`qm set <vmid> --args "-machine sata=off"`) and tell me so I can look for an
+API-safe alternative.
+
+## 6. oswl-connector: the headscale side
+
+`oswl-connector` (goose, vmid 212, 192.168.7.62) runs tailscaled in a container
+against `https://hs.atlashorizon.net`, advertising `192.168.7.0/24` and itself
+as an exit node. Three things have to happen outside this repo:
+
+1. **Create the preauth key and put it in Infisical** as `HEADSCALE_AUTHKEY`
+   under `/terraform` (it is read by `secrets.tmpl` as `headscale_authkey`).
+   It must be both ephemeral and reusable -- the VM keeps tailscaled's state in
+   memory, so it re-registers on every boot:
+
+   ```bash
+   headscale preauthkeys create --user <user> --ephemeral --reusable --expiration 8760h
+   ```
+
+   Until that secret exists, `tofu plan` fails while rendering
+   `butane/oswl_connector.yaml.tftpl` (`templatefile`: no value for
+   "headscale_authkey").
+
+2. **Approve the route and the exit node** once the VM has registered --
+   headscale honours neither until enabled:
+
+   ```bash
+   headscale nodes list
+   headscale nodes approve-routes -i <id> -r 192.168.7.0/24,0.0.0.0/0,::/0
+   ```
+
+   (On headscale older than 0.26 that is `headscale routes list` plus
+   `headscale routes enable -r <route-id>`, once per route.)
+
+   Because the node is ephemeral it re-registers on every boot, so this has to
+   be redone after each reboot and after each `tofu taint` rebuild. Headscale
+   can do it automatically, but **the policy has to exist before the node
+   advertises the tag** -- `tailscale up --advertise-tags` fails outright if
+   the tag is not owned by the registering user, which takes tailscaled down
+   with it. So: add the policy first, confirm it loads, and only then add
+   `--advertise-tags=tag:oswl-connector` to `TS_EXTRA_ARGS` in
+   `butane/oswl_connector.yaml.tftpl` and rebuild the VM.
+
+   ```json
+   {
+     "tagOwners": {
+       "tag:oswl-connector": ["<user>@"]
+     },
+     "autoApprovers": {
+       "routes": {
+         "192.168.7.0/24": ["tag:oswl-connector"]
+       },
+       "exitNode": ["tag:oswl-connector"]
+     }
+   }
+   ```
+
+   `<user>@` is the user the preauth key belongs to (policy v2 spells
+   usernames with a trailing `@`). Load the policy with `policy.mode: file` +
+   `policy.path` in headscale's `config.yaml`, or `headscale policy set -f
+   policy.json` in database mode.
+
+   Two further caveats if you go that route:
+
+   - **List the tag in `autoApprovers`, not a user or a group.** Approval by
+     username or group silently does not fire for a node whose identity is a
+     tag ([#2891](https://github.com/juanfont/headscale/issues/2891), open as
+     of 0.27.1). Naming the tag is both the documented form and the workaround.
+   - **A tagged node is governed by the tag in ACLs, not by its user.** With
+     headscale's default allow-all policy that changes nothing, but if you
+     later write real ACL rules, `tag:oswl-connector` needs its own grants.
+
+3. **Check the LAN's return path.** The VM has one NIC on VLAN 7, so clients
+   reaching 192.168.7.0/24 through the tunnel arrive with a tailnet source
+   address. The container runs with `--network host` and tailscaled installs
+   its own SNAT rules by default (`--snat-subnet-routes`), so this works
+   without touching the router; drop that default only if you add a route for
+   the tailnet CIDR on the 192.168.7.0/24 gateway.
+
+Note the same caveat as the Gitea runner: the auth key ends up in plaintext in
+the rendered Ignition snippet under `/var/lib/vz/snippets` on goose.
+
+## 7. Commit
+
+Nothing is committed. The `opentofu` submodule has the module changes; commit
+and push it first, then commit the parent repo (which records the new
+submodule pointer plus `providers.tf`, `vars.tf`, `main.tf`,
+`terraform.tfvars`, `secrets.tmpl`, `butane/`, and this file).

main.tf

@@ -11,7 +11,8 @@ locals {
   all_declared_hosts = flatten([
     for hv_name, hv in var.nodes : concat(
       [for name, lxc in try(hv.lxc, {}) : { name = name, ip = split("/", lxc.ip)[0] }],
-      [for name, vm in try(hv.machines, {}) : { name = name, ip = split("/", vm.ip)[0] }]
+      [for name, vm in try(hv.machines, {}) : { name = name, ip = split("/", vm.ip)[0] }],
+      [for name, fc in try(hv.flatcar, {}) : { name = name, ip = fc.ip }]
     )
   ])
 
@@ -38,16 +39,20 @@ locals {
 module "proxmox" {
   source = "./opentofu/modules/proxmox_node"
 
-  node_name       = var.hypervisors["proxmox"].node_name
+  node_name            = var.hypervisors["proxmox"].node_name
-  storage         = var.hypervisors["proxmox"].storage
+  storage              = var.hypervisors["proxmox"].storage
-  lxc             = try(var.nodes["proxmox"].lxc, {})
+  lxc                  = try(var.nodes["proxmox"].lxc, {})
-  machines        = try(var.nodes["proxmox"].machines, {})
+  machines             = try(var.nodes["proxmox"].machines, {})
-  search_domain   = local.node_common.search_domain
+  flatcar              = try(var.nodes["proxmox"].flatcar, {})
-  dns_nameservers = local.node_common.dns_nameservers
+  search_domain        = local.node_common.search_domain
-  ssh_key         = local.node_common.ssh_key
+  dns_nameservers      = local.node_common.dns_nameservers
+  ssh_key              = local.node_common.ssh_key
   ansible_user         = local.node_common.ansible_user
   lxc_password         = local.node_common.lxc_password
   ssh_private_key_path = local.node_common.ssh_private_key_path
+  flatcar_image        = var.flatcar_image
+  snippets_datastore   = var.snippets_datastore
+  butane_vars          = data.external.infisical.result
 
   providers = {
     proxmox = proxmox.proxmox
@@ -59,16 +64,20 @@ module "proxmox" {
 module "goose" {
   source = "./opentofu/modules/proxmox_node"
 
-  node_name       = var.hypervisors["goose"].node_name
+  node_name            = var.hypervisors["goose"].node_name
-  storage         = var.hypervisors["goose"].storage
+  storage              = var.hypervisors["goose"].storage
-  lxc             = try(var.nodes["goose"].lxc, {})
+  lxc                  = try(var.nodes["goose"].lxc, {})
-  machines        = try(var.nodes["goose"].machines, {})
+  machines             = try(var.nodes["goose"].machines, {})
-  search_domain   = local.node_common.search_domain
+  flatcar              = try(var.nodes["goose"].flatcar, {})
-  dns_nameservers = local.node_common.dns_nameservers
+  search_domain        = local.node_common.search_domain
-  ssh_key         = local.node_common.ssh_key
+  dns_nameservers      = local.node_common.dns_nameservers
+  ssh_key              = local.node_common.ssh_key
   ansible_user         = local.node_common.ansible_user
   lxc_password         = local.node_common.lxc_password
   ssh_private_key_path = local.node_common.ssh_private_key_path
+  flatcar_image        = var.flatcar_image
+  snippets_datastore   = var.snippets_datastore
+  butane_vars          = data.external.infisical.result
 
   providers = {
     proxmox = proxmox.goose
@@ -77,13 +86,13 @@ module "goose" {
 
 locals {
   hv_hosts = {
-    proxmox = { lxc = module.proxmox.lxc_hosts, vm = module.proxmox.vm_hosts }
+    proxmox = { lxc = module.proxmox.lxc_hosts, vm = module.proxmox.vm_hosts, flatcar = module.proxmox.flatcar_hosts }
-    goose    = { lxc = module.goose.lxc_hosts, vm = module.goose.vm_hosts }
+    goose   = { lxc = module.goose.lxc_hosts, vm = module.goose.vm_hosts, flatcar = module.goose.flatcar_hosts }
   }
 
   all_hosts = flatten([
     for hv_name, hv in local.hv_hosts : [
-      for h in concat(hv.lxc, hv.vm) : [
+      for h in concat(hv.lxc, hv.vm, hv.flatcar) : [
         for tag in h.filtered_tags : {
           tag  = tag
           name = h.name
@@ -138,7 +147,7 @@ module "caddy" {
 
 resource "local_file" "ansible_inventory" {
   filename = "${path.module}/${var.ansible_inventory_path}"
-  content  = templatefile("${path.module}/opentofu/inventory.tpl", {
+  content = templatefile("${path.module}/opentofu/inventory.tpl", {
     hypervisors                  = local.hv_hosts
     tag_groups                   = local.tag_groups
     ansible_user                 = var.ansible_user

providers.tf

@@ -8,6 +8,11 @@ terraform {
       source  = "hashicorp/local"
       version = "~> 2.2"
     }
+    # Butane -> Ignition transpiler for Flatcar VMs (no provider config needed).
+    ct = {
+      source  = "poseidon/ct"
+      version = "~> 0.14"
+    }
     technitium = {
       source  = "kenske/technitium"
       version = "~> 0.2"
@@ -71,6 +76,19 @@ provider "proxmox" {
   endpoint  = trimsuffix(var.hypervisors["proxmox"].api_url, "/api2/json")
   api_token = "${data.external.infisical.result["token_id"]}=${data.external.infisical.result["hv2_token_secret"]}"
   insecure  = true
+
+  # SSH is only used to upload Ignition snippets for Flatcar VMs (Proxmox has
+  # no API for snippets). Requires the ansible public key in root's
+  # authorized_keys on the node; see flatcar-todo.md.
+  ssh {
+    username    = "root"
+    private_key = base64decode(data.external.infisical.result["ssh_private_key_b64"])
+
+    node {
+      name    = var.hypervisors["proxmox"].node_name
+      address = regex("^https?://([^:/]+)", var.hypervisors["proxmox"].api_url)[0]
+    }
+  }
 }
 
 provider "proxmox" {
@@ -78,4 +96,17 @@ provider "proxmox" {
   endpoint  = trimsuffix(var.hypervisors["goose"].api_url, "/api2/json")
   api_token = "${data.external.infisical.result["token_id"]}=${data.external.infisical.result["goose_token_secret"]}"
   insecure  = true
+
+  # SSH is only used to upload Ignition snippets for Flatcar VMs (Proxmox has
+  # no API for snippets). Requires the ansible public key in root's
+  # authorized_keys on the node; see flatcar-todo.md.
+  ssh {
+    username    = "root"
+    private_key = base64decode(data.external.infisical.result["ssh_private_key_b64"])
+
+    node {
+      name    = var.hypervisors["goose"].node_name
+      address = regex("^https?://([^:/]+)", var.hypervisors["goose"].api_url)[0]
+    }
+  }
 }

secrets.tmpl

@@ -10,6 +10,11 @@
   {{- if eq $secret.Key "IPAM_USERNAME" }}"ipam_username": "{{$secret.Value}}",{{ end }}
   {{- if eq $secret.Key "IPAM_PASSWORD" }}"ipam_password": "{{$secret.Value}}",{{ end }}
   {{- if eq $secret.Key "TECHNITIUM_API_TOKEN" }}"technitium_api_token": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "GITEA_RUNNER_TOKEN" }}"gitea_runner_token": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "LITELLM_DATABASE_URL" }}"litellm_database_url": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "LITELLM_MASTER_KEY" }}"litellm_master_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "LITELLM_SALT_KEY" }}"litellm_salt_key": "{{$secret.Value}}",{{ end }}
+  {{- if eq $secret.Key "HEADSCALE_AUTHKEY" }}"headscale_authkey": "{{$secret.Value}}",{{ end }}
   {{- end }}
   "_sentinel": "1"
 }

terraform.tfvars

@@ -195,28 +195,50 @@ nodes = {
         gw           = "192.168.0.1"
       }
     }
-    machines = {
+    # Flatcar Container Linux VMs, provisioned by Ignition. Read
+    # opentofu/README.md (Flatcar section) and flatcar-todo.md before enabling.
+    flatcar = {
       "oswald-runner" = {
-        hostname   = "oswald-runner"
+        hostname         = "oswald-runner"
-        vmid       = 202
+        vmid             = 210
-        ip         = "192.168.7.50"
+        ip               = "192.168.7.60"
-        vlan       = 7
+        vlan             = 7
-        template   = "Debian13"
+        memory           = 4096
-        full_clone = true
+        cores            = 4
-        onboot     = true
+        disk_size        = 32
-        tags       = ""
+        qemu_guest_agent = false
-        agent      = 1
+        # Rendered with templatefile(): ${gitea_runner_token} comes from
-        memory     = 4096
+        # Infisical (GITEA_RUNNER_TOKEN in secrets.tmpl), so the token is
-        disk_size  = "32"
+        # never committed.
-        balloon    = 1
+        butane_file = "butane/oswald_runner.yaml.tftpl"
-        cpu_type   = "kvm64"
-        cores      = 4
-        sockets    = 1
-        vcpus      = 0
-        bios       = "ovmf"
-        machine    = "q35"
-        bridge     = "vmbr0"
       }
+      "ai" = {
+        hostname         = "ai"
+        vmid             = 211
+        ip               = "192.168.7.61"
+        vlan             = 7
+        memory           = 4096
+        cores            = 2
+        disk_size        = 32
+        qemu_guest_agent = false
+        butane_file      = "butane/ai.yaml.tftpl"
+      }
+      # Headscale subnet router / exit node. Registers as an ephemeral node,
+      # so the preauth key must be --ephemeral --reusable; the routes and the
+      # exit node still need approving on the headscale side once it appears.
+      "oswl-connector" = {
+        hostname         = "oswl-connector"
+        vmid             = 212
+        ip               = "192.168.7.62"
+        vlan             = 7
+        memory           = 2048
+        cores            = 2
+        disk_size        = 32
+        qemu_guest_agent = false
+        butane_file      = "butane/oswl_connector.yaml.tftpl"
+      }
+    }
+    machines = {
       "grist" = {
         hostname   = "grist"
         vmid       = 204

vars.tf

@@ -103,9 +103,9 @@ variable "ansible_inventory_path" {
   default     = "./ansible/inventories/generated.yml"
 }
 
-# LXC and VM definitions per hypervisor, keyed by the same names used in
+# LXC, VM and Flatcar VM definitions per hypervisor, keyed by the same names
-# hypervisors and providers.tf. Both lxc and machines default to {} so a node
+# used in hypervisors and providers.tf. All three maps default to {} so a node
-# can be declared with only one type of workload.
+# can be declared with any subset of workload types.
 variable "nodes" {
   type = map(object({
     lxc = optional(map(object({
@@ -162,6 +162,44 @@ variable "nodes" {
         gw     = optional(string)
       }))
     })), {})
+    flatcar = optional(map(object({
+      hostname         = string
+      vmid             = number
+      ip               = string
+      gw               = optional(string)
+      vlan             = optional(number, 0)
+      bridge           = optional(string, "vmbr0")
+      onboot           = optional(bool, true)
+      tags             = optional(string, "")
+      cores            = optional(number, 2)
+      cpu_type         = optional(string, "x86-64-v2-AES")
+      memory           = optional(number, 4096)
+      disk_size        = optional(number, 32)
+      qemu_guest_agent = optional(bool, false) # needs a Butane unit running qemu-ga in a container
+      butane           = optional(string)      # inline Butane YAML
+      butane_file      = optional(string)      # path relative to this directory
+    })), {})
   }))
   default = {}
 }
+
+# Flatcar release imported onto every node that runs Flatcar VMs. Pin a version
+# and copy its sha512 from the image's .DIGESTS file; see opentofu/README.md.
+variable "flatcar_image" {
+  type = object({
+    channel   = optional(string, "stable")
+    version   = string
+    sha512    = string
+    datastore = optional(string, "local")
+  })
+  default = {
+    version = "4593.2.5"
+    sha512  = "21fdba07ffc73aac80aa40f1f99e0460e2d53b1c2815a774d5e2940d2b5cfa78fb2c86391fbf3bb76e28e8ece1ca41333f30ce18d1dd3f0c016e95bed120aba8"
+  }
+}
+
+variable "snippets_datastore" {
+  description = "Datastore holding Ignition snippets for Flatcar VMs. Needs the snippets content type enabled on every node."
+  type        = string
+  default     = "local"
+}