diff --git a/butane/ai.yaml.tftpl b/butane/ai.yaml.tftpl new file mode 100644 index 0000000..77ad6af --- /dev/null +++ b/butane/ai.yaml.tftpl @@ -0,0 +1,47 @@ +variant: flatcar +version: 1.0.0 + +systemd: + units: + - name: gitea-mcp.service + enabled: true + contents: | + [Unit] + Description=Gitea MCP Server + After=docker.service + Requires=docker.service + [Service] + TimeoutStartSec=0 + ExecStartPre=-/usr/bin/docker rm --force gitea-mcp + ExecStart=/usr/bin/docker run --name gitea-mcp \ + -e GITEA_HOST=https://git.atlashorizon.net \ + -p 8080:8080 \ + docker.gitea.com/gitea-mcp-server \ + /app/gitea-mcp -t http --port 8080 + ExecStop=/usr/bin/docker stop gitea-mcp + Restart=always + RestartSec=5s + [Install] + WantedBy=multi-user.target + - name: litellm.service + enabled: true + contents: | + [Unit] + Description=LiteLLM proxy + After=docker.service + Requires=docker.service + [Service] + TimeoutStartSec=0 + ExecStartPre=-/usr/bin/docker rm --force litellm + ExecStart=/usr/bin/docker run --name litellm \ + -e DATABASE_URL=${litellm_database_url} \ + -e STORE_MODEL_IN_DB=True \ + -e LITELLM_MASTER_KEY=${litellm_master_key} \ + -e LITELLM_SALT_KEY=${litellm_salt_key} \ + -p 4000:4000 \ + docker.litellm.ai/berriai/litellm:main-stable + ExecStop=/usr/bin/docker stop litellm + Restart=always + RestartSec=5s + [Install] + WantedBy=multi-user.target diff --git a/butane/example.yaml b/butane/example.yaml new file mode 100644 index 0000000..6335c57 --- /dev/null +++ b/butane/example.yaml @@ -0,0 +1,36 @@ +# Example extra Butane config for a Flatcar VM. Reference it from tfvars with +# `butane_file = "butane/.yaml"`. It is merged into the base config the +# module generates (hostname, core + ansible users with the shared SSH key). +# Applied by Ignition on FIRST BOOT only: to change a running VM, edit this +# file and taint the VM so it is rebuilt from the image. +variant: flatcar +version: 1.0.0 + +systemd: + units: + # Example: run a container as a service. + - name: hello.service + enabled: true + contents: | + [Unit] + Description=Hello container + After=docker.service + Requires=docker.service + [Service] + TimeoutStartSec=0 + ExecStartPre=-/usr/bin/docker rm --force hello + ExecStart=/usr/bin/docker run --name hello --rm -p 8080:80 nginx:alpine + ExecStop=/usr/bin/docker stop hello + Restart=always + RestartSec=5s + [Install] + WantedBy=multi-user.target + +storage: + files: + # Example: a file on disk. + - path: /etc/motd + mode: 0644 + contents: + inline: | + Managed by OpenTofu + Ignition. Config changes require a rebuild. diff --git a/butane/oswald_runner.yaml.tftpl b/butane/oswald_runner.yaml.tftpl new file mode 100644 index 0000000..53eba13 --- /dev/null +++ b/butane/oswald_runner.yaml.tftpl @@ -0,0 +1,49 @@ +variant: flatcar +version: 1.0.0 + +systemd: + units: + # Gitea Actions runner. Registers once on first start and keeps its + # identity (the .runner file) in /var/lib/oswald_runner on the VM's own + # root disk -- not ephemeral, so it does not self-delete after each job + # (that caused stale offline entries whenever the container was + # interrupted before finishing a job; see flatcar-todo.md). Reusing the + # same registration also means the token never needs rotating. + # + # /var/lib/oswald_runner survives container restarts and VM reboots (it's + # on disk, not in the container), but not a VM rebuild (taint): that + # recreates the disk from the image, so the runner re-registers fresh + # next boot. Old entries from before this change, or from any future + # rebuild, still need manual cleanup in Gitea's runner admin page. + # + # GITEA_RUNNER_REGISTRATION_TOKEN is templated in from Infisical + # (GITEA_RUNNER_TOKEN in secrets.tmpl) by main.tf, not stored here. It + # still ends up in plaintext in the rendered Ignition snippet on the + # node's storage -- see the caveat in flatcar-todo.md. + - name: oswald_runner.service + enabled: true + contents: | + [Unit] + Description=Gitea Actions runner + After=docker.service + Requires=docker.service + [Service] + TimeoutStartSec=0 + ExecStartPre=-/usr/bin/docker rm --force oswald_runner + ExecStart=/usr/bin/docker run --name oswald_runner \ + -e GITEA_INSTANCE_URL=https://git.atlashorizon.net \ + -e GITEA_RUNNER_REGISTRATION_TOKEN=${gitea_runner_token} \ + -e GITEA_RUNNER_NAME=oswald_runner \ + -v /var/lib/oswald_runner:/data \ + -v /var/run/docker.sock:/var/run/docker.sock \ + docker.io/gitea/runner:3 + ExecStop=/usr/bin/docker stop oswald_runner + Restart=always + RestartSec=5s + [Install] + WantedBy=multi-user.target + +storage: + directories: + - path: /var/lib/oswald_runner + mode: 0755 diff --git a/butane/oswl_connector.yaml.tftpl b/butane/oswl_connector.yaml.tftpl new file mode 100644 index 0000000..480101e --- /dev/null +++ b/butane/oswl_connector.yaml.tftpl @@ -0,0 +1,60 @@ +variant: flatcar +version: 1.0.0 + +# Headscale (Tailscale) node: advertises 192.168.7.0/24 as a subnet route and +# offers itself as an exit node. +# +# Registered as an EPHEMERAL node: tailscaled runs with in-memory state, so the +# VM re-authenticates on every boot and headscale drops the node shortly after +# it goes offline. That requires the preauth key to be both --ephemeral and +# --reusable, otherwise the second boot fails to register. +# +# HEADSCALE_AUTHKEY is templated in from Infisical (see secrets.tmpl) by +# main.tf, not stored here. It still ends up in plaintext in the rendered +# Ignition snippet on the node's storage -- see the caveat in flatcar-todo.md. + +systemd: + units: + - name: tailscaled.service + enabled: true + contents: | + [Unit] + Description=Tailscale node (headscale subnet router + exit node) + After=docker.service network-online.target + Requires=docker.service + Wants=network-online.target + [Service] + TimeoutStartSec=0 + # tun is not loaded by default on a stock Flatcar boot. + ExecStartPre=-/usr/sbin/modprobe tun + # Recommended by Tailscale for subnet routers and exit nodes: without + # this, UDP throughput through the node is roughly halved. + ExecStartPre=-/usr/bin/bash -c 'ethtool -K $(ip route show 0.0.0.0/0 | cut -d" " -f5) rx-udp-gro-forwarding on rx-gro-list off' + ExecStartPre=-/usr/bin/docker rm --force tailscaled + ExecStart=/usr/bin/docker run --name tailscaled \ + --network host --cap-add NET_ADMIN --cap-add NET_RAW \ + --device /dev/net/tun \ + -e TS_AUTHKEY=${headscale_authkey} \ + -e TS_HOSTNAME=oswl-connector \ + -e TS_ROUTES=192.168.7.0/24 \ + -e TS_ACCEPT_DNS=false \ + -e "TS_EXTRA_ARGS=--login-server=https://hs.atlashorizon.net --advertise-exit-node" \ + -e TS_USERSPACE=false \ + docker.io/tailscale/tailscale:stable + ExecStop=/usr/bin/docker stop tailscaled + Restart=always + RestartSec=5s + [Install] + WantedBy=multi-user.target + +storage: + files: + # Forwarding must be on for the subnet route and the exit node to pass + # traffic at all; the container shares the host netns (--network host), so + # this has to be set on the host, not inside it. + - path: /etc/sysctl.d/99-tailscale.conf + mode: 0644 + contents: + inline: | + net.ipv4.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 diff --git a/butane/webserver.yaml b/butane/webserver.yaml new file mode 100644 index 0000000..736cf2d --- /dev/null +++ b/butane/webserver.yaml @@ -0,0 +1,75 @@ +# Butane config for a Flatcar VM running the official httpd (Apache) image +# as a docker container. Applied by Ignition on FIRST BOOT only: to change a +# running VM, edit this file and taint the VM so it is rebuilt from the image. +variant: flatcar +version: 1.0.0 + +systemd: + units: + - name: webserver.service + enabled: true + contents: | + [Unit] + Description=httpd webserver container + After=docker.service + Requires=docker.service + [Service] + TimeoutStartSec=0 + ExecStartPre=-/usr/bin/docker rm --force webserver + ExecStart=/usr/bin/docker run --name webserver --rm -p 80:80 httpd:latest + ExecStop=/usr/bin/docker stop webserver + Restart=always + RestartSec=5s + [Install] + WantedBy=multi-user.target + # Experimental: Flatcar ships no qemu-guest-agent binary, so this runs one + # from an Alpine container against the virtio-serial channel Proxmox + # exposes when the VM's `agent.enabled` is true (see qemu_guest_agent in + # tfvars). --network host is required so qemu-ga reports the host's real + # network interfaces/IP instead of the container's own bridge IP. + # --pid host + the /sbin/shutdown bind mount (see /opt/qga-shutdown.sh + # below) are required so guest-shutdown/reboot actually reach the host + # instead of Alpine's own busybox init inside the container. + # --privileged + host /dev is broader access than qemu-ga strictly needs, + # but keeps this simple for a first try. + - name: qemu-guest-agent.service + enabled: true + contents: | + [Unit] + Description=QEMU guest agent (containerized) + After=docker.service + Requires=docker.service + [Service] + TimeoutStartSec=0 + ExecStartPre=-/usr/bin/docker rm --force qemu-guest-agent + ExecStart=/usr/bin/docker run --name qemu-guest-agent --rm --privileged \ + --network host --pid host -v /dev:/dev \ + -v /opt/qga-shutdown.sh:/sbin/shutdown:ro \ + alpine:latest sh -c \ + "apk add --no-cache qemu-guest-agent util-linux && exec qemu-ga -m virtio-serial -p /dev/virtio-ports/org.qemu.guest_agent.0" + Restart=always + RestartSec=5s + [Install] + WantedBy=multi-user.target + +storage: + files: + # qemu-ga always execs "/sbin/shutdown +0 " (flag is -H + # halt, -P powerdown, -r reboot) to handle guest-shutdown/guest-reboot. + # Inside the container that path is Alpine's own busybox shutdown, which + # only affects the container's namespace. Bind-mounted over /sbin/shutdown + # in the container (see qemu-guest-agent.service above), this shim uses + # nsenter to re-enter the real host's namespaces (PID 1 in the container + # is the host's actual systemd, since the container runs with --pid host) + # and calls the host's real systemctl instead. + - path: /opt/qga-shutdown.sh + mode: 0755 + contents: + inline: | + #!/bin/sh + case "$1" in + -H) action=halt ;; + -r) action=reboot ;; + *) action=poweroff ;; + esac + exec nsenter --target 1 --mount --uts --ipc --net -- systemctl "$action" diff --git a/flatcar-todo.md b/flatcar-todo.md new file mode 100644 index 0000000..79f7a75 --- /dev/null +++ b/flatcar-todo.md @@ -0,0 +1,187 @@ +# Flatcar on Proxmox: what is left for you + +The OpenTofu side is done: `nodes..flatcar` in `terraform.tfvars` now creates +Flatcar VMs (see the Flatcar section in `opentofu/README.md` for how it works). +The items below need access or decisions I do not have. Nothing in this list +touches the existing LXCs/VMs. + +## 1. Enable the Snippets content type on `local` (both nodes) + +`local` already has `import` enabled (needed for the image), but not `snippets` +(needed for the Ignition file). On each node, as root: + +```bash +pvesm set local --content backup,vztmpl,import,iso,snippets +``` + +Or in the UI: Datacenter → Storage → local → Edit → Content → tick Snippets. + +Proxmox does **not** create the snippets directory when you enable the content +type -- you have to do it yourself, or the upload fails with `tee: +/var/lib/vz/snippets/-ignition.json: No such file or directory`: + +```bash +mkdir -p /var/lib/vz/snippets # confirm the path via `local`'s entry in /etc/pve/storage.cfg +``` + +## 2. Let the provider SSH to the nodes as root (both nodes) + +Proxmox has no API for uploading snippets, so bpg copies them over SSH. The +provider is configured (`providers.tf`, `ssh` block) to log in as `root` with +the ansible key from Infisical (`SSH_PRIVATE_KEY` / `SSH_PUBLIC_KEY`). On each +node append the public key to root's authorized_keys: + +```bash +# from this machine; the key file is materialized by tofu at ~/.ssh/infra_ansible_key +ssh-copy-id -i ~/.ssh/infra_ansible_key.pub root@192.168.7.240 # pve +ssh-copy-id -i ~/.ssh/infra_ansible_key.pub root@192.168.7.241 # goose +``` + +(`~/.ssh/infra_ansible_key.pub` may not exist yet; paste `SSH_PUBLIC_KEY` from +Infisical into `/root/.ssh/authorized_keys` on the node instead.) + +Then verify from here: + +```bash +ssh -i ~/.ssh/infra_ansible_key root@192.168.7.241 'pvesm status' +``` + +If you would rather not give the provider root, bpg supports a non-root user +with a narrow sudoers file (`pvesm`, `qm`, `tee /var/lib/vz/snippets/*`). Change +`username` in the `ssh` blocks accordingly. + +## 3. Declare a VM + +Uncomment and adapt the `flatcar = { ... }` example under `nodes.goose` in +`terraform.tfvars`. Pick an unused vmid and IP. Optional per-VM config goes in +`butane/.yaml`, referenced with `butane_file = "butane/.yaml"` on +the VM (`butane/example.yaml` is a starting point; keep `variant: flatcar` and +`version: 1.0.0`). Inline YAML via `butane = <<-EOT ... EOT` also works. + +I dry-ran a plan with the example VM declared (nothing applied): it creates +the image download, the Ignition snippet, the VM and its DNS record, and +changes nothing else. + +Then: + +```bash +infisical run --path /terraform -- tofu plan -out plan +infisical run --path /terraform -- tofu apply plan +``` + +The first apply on a node downloads the 500 MB Flatcar image into `local` +(`flatcar-stable-4593.2.5.img`), then creates the VM with that image imported +as its boot disk. Expect the download to take a minute or two. + +Log in with `ssh ansible@` or `ssh core@` using the ansible key. + +## 4. Things to know + +- Ignition runs on first boot only. Editing the Butane file updates the snippet on the + node but does nothing to a running VM. To apply, rebuild it: + `tofu taint 'module.goose.proxmox_virtual_environment_vm.flatcar[""]'` + (this destroys the VM's disk; keep data in volumes you can recreate). +- Ignition snippets are plain text on the node. Do not put secrets in `butane`. +- Flatcar has no Python, so the existing Ansible plays will not run against it. + These hosts land in a `flatcar` group in the generated inventory; only + `raw`/`script` tasks work there. Ignition is the intended replacement. +- No QEMU guest agent: the module sets `agent.enabled = false` for Flatcar VMs, + so plans never wait on it. Proxmox will not show the VM's IP in the UI. +- Bumping `flatcar_image.version` (in `vars.tf` default or tfvars) plus its + sha512 from the release `.DIGESTS` file downloads a new image for future VMs + only; existing VMs are not touched (`import_from` is ignored after creation). + Flatcar itself auto-updates in place via its update engine. + +## 5. If the VM does not boot + +The Flatcar docs pass `--args "-machine sata=off"` when creating the VM. I left +it out: that option is root-only in Proxmox and the API token cannot set it. +If a Flatcar VM hangs at boot, check the serial console (`qm terminal `) +first; if it looks firmware-related, set that arg once by hand as root +(`qm set --args "-machine sata=off"`) and tell me so I can look for an +API-safe alternative. + +## 6. oswl-connector: the headscale side + +`oswl-connector` (goose, vmid 212, 192.168.7.62) runs tailscaled in a container +against `https://hs.atlashorizon.net`, advertising `192.168.7.0/24` and itself +as an exit node. Three things have to happen outside this repo: + +1. **Create the preauth key and put it in Infisical** as `HEADSCALE_AUTHKEY` + under `/terraform` (it is read by `secrets.tmpl` as `headscale_authkey`). + It must be both ephemeral and reusable -- the VM keeps tailscaled's state in + memory, so it re-registers on every boot: + + ```bash + headscale preauthkeys create --user --ephemeral --reusable --expiration 8760h + ``` + + Until that secret exists, `tofu plan` fails while rendering + `butane/oswl_connector.yaml.tftpl` (`templatefile`: no value for + "headscale_authkey"). + +2. **Approve the route and the exit node** once the VM has registered -- + headscale honours neither until enabled: + + ```bash + headscale nodes list + headscale nodes approve-routes -i -r 192.168.7.0/24,0.0.0.0/0,::/0 + ``` + + (On headscale older than 0.26 that is `headscale routes list` plus + `headscale routes enable -r `, once per route.) + + Because the node is ephemeral it re-registers on every boot, so this has to + be redone after each reboot and after each `tofu taint` rebuild. Headscale + can do it automatically, but **the policy has to exist before the node + advertises the tag** -- `tailscale up --advertise-tags` fails outright if + the tag is not owned by the registering user, which takes tailscaled down + with it. So: add the policy first, confirm it loads, and only then add + `--advertise-tags=tag:oswl-connector` to `TS_EXTRA_ARGS` in + `butane/oswl_connector.yaml.tftpl` and rebuild the VM. + + ```json + { + "tagOwners": { + "tag:oswl-connector": ["@"] + }, + "autoApprovers": { + "routes": { + "192.168.7.0/24": ["tag:oswl-connector"] + }, + "exitNode": ["tag:oswl-connector"] + } + } + ``` + + `@` is the user the preauth key belongs to (policy v2 spells + usernames with a trailing `@`). Load the policy with `policy.mode: file` + + `policy.path` in headscale's `config.yaml`, or `headscale policy set -f + policy.json` in database mode. + + Two further caveats if you go that route: + + - **List the tag in `autoApprovers`, not a user or a group.** Approval by + username or group silently does not fire for a node whose identity is a + tag ([#2891](https://github.com/juanfont/headscale/issues/2891), open as + of 0.27.1). Naming the tag is both the documented form and the workaround. + - **A tagged node is governed by the tag in ACLs, not by its user.** With + headscale's default allow-all policy that changes nothing, but if you + later write real ACL rules, `tag:oswl-connector` needs its own grants. + +3. **Check the LAN's return path.** The VM has one NIC on VLAN 7, so clients + reaching 192.168.7.0/24 through the tunnel arrive with a tailnet source + address. The container runs with `--network host` and tailscaled installs + its own SNAT rules by default (`--snat-subnet-routes`), so this works + without touching the router; drop that default only if you add a route for + the tailnet CIDR on the 192.168.7.0/24 gateway. + +Note the same caveat as the Gitea runner: the auth key ends up in plaintext in +the rendered Ignition snippet under `/var/lib/vz/snippets` on goose. + +## 7. Commit + +Nothing is committed. The `opentofu` submodule has the module changes; commit +and push it first, then commit the parent repo (which records the new +submodule pointer plus `providers.tf`, `vars.tf`, `main.tf`, +`terraform.tfvars`, `secrets.tmpl`, `butane/`, and this file). diff --git a/main.tf b/main.tf index 99fabc4..64fb6ab 100644 --- a/main.tf +++ b/main.tf @@ -11,7 +11,8 @@ locals { all_declared_hosts = flatten([ for hv_name, hv in var.nodes : concat( [for name, lxc in try(hv.lxc, {}) : { name = name, ip = split("/", lxc.ip)[0] }], - [for name, vm in try(hv.machines, {}) : { name = name, ip = split("/", vm.ip)[0] }] + [for name, vm in try(hv.machines, {}) : { name = name, ip = split("/", vm.ip)[0] }], + [for name, fc in try(hv.flatcar, {}) : { name = name, ip = fc.ip }] ) ]) @@ -38,16 +39,20 @@ locals { module "proxmox" { source = "./opentofu/modules/proxmox_node" - node_name = var.hypervisors["proxmox"].node_name - storage = var.hypervisors["proxmox"].storage - lxc = try(var.nodes["proxmox"].lxc, {}) - machines = try(var.nodes["proxmox"].machines, {}) - search_domain = local.node_common.search_domain - dns_nameservers = local.node_common.dns_nameservers - ssh_key = local.node_common.ssh_key + node_name = var.hypervisors["proxmox"].node_name + storage = var.hypervisors["proxmox"].storage + lxc = try(var.nodes["proxmox"].lxc, {}) + machines = try(var.nodes["proxmox"].machines, {}) + flatcar = try(var.nodes["proxmox"].flatcar, {}) + search_domain = local.node_common.search_domain + dns_nameservers = local.node_common.dns_nameservers + ssh_key = local.node_common.ssh_key ansible_user = local.node_common.ansible_user lxc_password = local.node_common.lxc_password ssh_private_key_path = local.node_common.ssh_private_key_path + flatcar_image = var.flatcar_image + snippets_datastore = var.snippets_datastore + butane_vars = data.external.infisical.result providers = { proxmox = proxmox.proxmox @@ -59,16 +64,20 @@ module "proxmox" { module "goose" { source = "./opentofu/modules/proxmox_node" - node_name = var.hypervisors["goose"].node_name - storage = var.hypervisors["goose"].storage - lxc = try(var.nodes["goose"].lxc, {}) - machines = try(var.nodes["goose"].machines, {}) - search_domain = local.node_common.search_domain - dns_nameservers = local.node_common.dns_nameservers - ssh_key = local.node_common.ssh_key + node_name = var.hypervisors["goose"].node_name + storage = var.hypervisors["goose"].storage + lxc = try(var.nodes["goose"].lxc, {}) + machines = try(var.nodes["goose"].machines, {}) + flatcar = try(var.nodes["goose"].flatcar, {}) + search_domain = local.node_common.search_domain + dns_nameservers = local.node_common.dns_nameservers + ssh_key = local.node_common.ssh_key ansible_user = local.node_common.ansible_user lxc_password = local.node_common.lxc_password ssh_private_key_path = local.node_common.ssh_private_key_path + flatcar_image = var.flatcar_image + snippets_datastore = var.snippets_datastore + butane_vars = data.external.infisical.result providers = { proxmox = proxmox.goose @@ -77,13 +86,13 @@ module "goose" { locals { hv_hosts = { - proxmox = { lxc = module.proxmox.lxc_hosts, vm = module.proxmox.vm_hosts } - goose = { lxc = module.goose.lxc_hosts, vm = module.goose.vm_hosts } + proxmox = { lxc = module.proxmox.lxc_hosts, vm = module.proxmox.vm_hosts, flatcar = module.proxmox.flatcar_hosts } + goose = { lxc = module.goose.lxc_hosts, vm = module.goose.vm_hosts, flatcar = module.goose.flatcar_hosts } } all_hosts = flatten([ for hv_name, hv in local.hv_hosts : [ - for h in concat(hv.lxc, hv.vm) : [ + for h in concat(hv.lxc, hv.vm, hv.flatcar) : [ for tag in h.filtered_tags : { tag = tag name = h.name @@ -138,7 +147,7 @@ module "caddy" { resource "local_file" "ansible_inventory" { filename = "${path.module}/${var.ansible_inventory_path}" - content = templatefile("${path.module}/opentofu/inventory.tpl", { + content = templatefile("${path.module}/opentofu/inventory.tpl", { hypervisors = local.hv_hosts tag_groups = local.tag_groups ansible_user = var.ansible_user diff --git a/providers.tf b/providers.tf index 02b5f2e..bf4f387 100644 --- a/providers.tf +++ b/providers.tf @@ -8,6 +8,11 @@ terraform { source = "hashicorp/local" version = "~> 2.2" } + # Butane -> Ignition transpiler for Flatcar VMs (no provider config needed). + ct = { + source = "poseidon/ct" + version = "~> 0.14" + } technitium = { source = "kenske/technitium" version = "~> 0.2" @@ -71,6 +76,19 @@ provider "proxmox" { endpoint = trimsuffix(var.hypervisors["proxmox"].api_url, "/api2/json") api_token = "${data.external.infisical.result["token_id"]}=${data.external.infisical.result["hv2_token_secret"]}" insecure = true + + # SSH is only used to upload Ignition snippets for Flatcar VMs (Proxmox has + # no API for snippets). Requires the ansible public key in root's + # authorized_keys on the node; see flatcar-todo.md. + ssh { + username = "root" + private_key = base64decode(data.external.infisical.result["ssh_private_key_b64"]) + + node { + name = var.hypervisors["proxmox"].node_name + address = regex("^https?://([^:/]+)", var.hypervisors["proxmox"].api_url)[0] + } + } } provider "proxmox" { @@ -78,4 +96,17 @@ provider "proxmox" { endpoint = trimsuffix(var.hypervisors["goose"].api_url, "/api2/json") api_token = "${data.external.infisical.result["token_id"]}=${data.external.infisical.result["goose_token_secret"]}" insecure = true + + # SSH is only used to upload Ignition snippets for Flatcar VMs (Proxmox has + # no API for snippets). Requires the ansible public key in root's + # authorized_keys on the node; see flatcar-todo.md. + ssh { + username = "root" + private_key = base64decode(data.external.infisical.result["ssh_private_key_b64"]) + + node { + name = var.hypervisors["goose"].node_name + address = regex("^https?://([^:/]+)", var.hypervisors["goose"].api_url)[0] + } + } } diff --git a/secrets.tmpl b/secrets.tmpl index fc34e11..fb17cf8 100644 --- a/secrets.tmpl +++ b/secrets.tmpl @@ -10,6 +10,11 @@ {{- if eq $secret.Key "IPAM_USERNAME" }}"ipam_username": "{{$secret.Value}}",{{ end }} {{- if eq $secret.Key "IPAM_PASSWORD" }}"ipam_password": "{{$secret.Value}}",{{ end }} {{- if eq $secret.Key "TECHNITIUM_API_TOKEN" }}"technitium_api_token": "{{$secret.Value}}",{{ end }} + {{- if eq $secret.Key "GITEA_RUNNER_TOKEN" }}"gitea_runner_token": "{{$secret.Value}}",{{ end }} + {{- if eq $secret.Key "LITELLM_DATABASE_URL" }}"litellm_database_url": "{{$secret.Value}}",{{ end }} + {{- if eq $secret.Key "LITELLM_MASTER_KEY" }}"litellm_master_key": "{{$secret.Value}}",{{ end }} + {{- if eq $secret.Key "LITELLM_SALT_KEY" }}"litellm_salt_key": "{{$secret.Value}}",{{ end }} + {{- if eq $secret.Key "HEADSCALE_AUTHKEY" }}"headscale_authkey": "{{$secret.Value}}",{{ end }} {{- end }} "_sentinel": "1" } diff --git a/terraform.tfvars b/terraform.tfvars index b40bd4e..686ed9c 100644 --- a/terraform.tfvars +++ b/terraform.tfvars @@ -195,28 +195,50 @@ nodes = { gw = "192.168.0.1" } } - machines = { + # Flatcar Container Linux VMs, provisioned by Ignition. Read + # opentofu/README.md (Flatcar section) and flatcar-todo.md before enabling. + flatcar = { "oswald-runner" = { - hostname = "oswald-runner" - vmid = 202 - ip = "192.168.7.50" - vlan = 7 - template = "Debian13" - full_clone = true - onboot = true - tags = "" - agent = 1 - memory = 4096 - disk_size = "32" - balloon = 1 - cpu_type = "kvm64" - cores = 4 - sockets = 1 - vcpus = 0 - bios = "ovmf" - machine = "q35" - bridge = "vmbr0" + hostname = "oswald-runner" + vmid = 210 + ip = "192.168.7.60" + vlan = 7 + memory = 4096 + cores = 4 + disk_size = 32 + qemu_guest_agent = false + # Rendered with templatefile(): ${gitea_runner_token} comes from + # Infisical (GITEA_RUNNER_TOKEN in secrets.tmpl), so the token is + # never committed. + butane_file = "butane/oswald_runner.yaml.tftpl" } + "ai" = { + hostname = "ai" + vmid = 211 + ip = "192.168.7.61" + vlan = 7 + memory = 4096 + cores = 2 + disk_size = 32 + qemu_guest_agent = false + butane_file = "butane/ai.yaml.tftpl" + } + # Headscale subnet router / exit node. Registers as an ephemeral node, + # so the preauth key must be --ephemeral --reusable; the routes and the + # exit node still need approving on the headscale side once it appears. + "oswl-connector" = { + hostname = "oswl-connector" + vmid = 212 + ip = "192.168.7.62" + vlan = 7 + memory = 2048 + cores = 2 + disk_size = 32 + qemu_guest_agent = false + butane_file = "butane/oswl_connector.yaml.tftpl" + } + } + machines = { "grist" = { hostname = "grist" vmid = 204 diff --git a/vars.tf b/vars.tf index 8bf1418..caa582b 100644 --- a/vars.tf +++ b/vars.tf @@ -103,9 +103,9 @@ variable "ansible_inventory_path" { default = "./ansible/inventories/generated.yml" } -# LXC and VM definitions per hypervisor, keyed by the same names used in -# hypervisors and providers.tf. Both lxc and machines default to {} so a node -# can be declared with only one type of workload. +# LXC, VM and Flatcar VM definitions per hypervisor, keyed by the same names +# used in hypervisors and providers.tf. All three maps default to {} so a node +# can be declared with any subset of workload types. variable "nodes" { type = map(object({ lxc = optional(map(object({ @@ -162,6 +162,44 @@ variable "nodes" { gw = optional(string) })) })), {}) + flatcar = optional(map(object({ + hostname = string + vmid = number + ip = string + gw = optional(string) + vlan = optional(number, 0) + bridge = optional(string, "vmbr0") + onboot = optional(bool, true) + tags = optional(string, "") + cores = optional(number, 2) + cpu_type = optional(string, "x86-64-v2-AES") + memory = optional(number, 4096) + disk_size = optional(number, 32) + qemu_guest_agent = optional(bool, false) # needs a Butane unit running qemu-ga in a container + butane = optional(string) # inline Butane YAML + butane_file = optional(string) # path relative to this directory + })), {}) })) default = {} } + +# Flatcar release imported onto every node that runs Flatcar VMs. Pin a version +# and copy its sha512 from the image's .DIGESTS file; see opentofu/README.md. +variable "flatcar_image" { + type = object({ + channel = optional(string, "stable") + version = string + sha512 = string + datastore = optional(string, "local") + }) + default = { + version = "4593.2.5" + sha512 = "21fdba07ffc73aac80aa40f1f99e0460e2d53b1c2815a774d5e2940d2b5cfa78fb2c86391fbf3bb76e28e8ece1ca41333f30ce18d1dd3f0c016e95bed120aba8" + } +} + +variable "snippets_datastore" { + description = "Datastore holding Ignition snippets for Flatcar VMs. Needs the snippets content type enabled on every node." + type = string + default = "local" +}