Updated project to improve guidance

terraform/cloud-init/agent.yaml.tpl

@@ -14,16 +14,30 @@ package_update: true
 packages:
   - curl
 
+# A script instead of a plain runcmd line for one reason: Terraform (see
+# main.tf's closing comment) brings all VMs up in parallel, with no
+# ordering guarantee that the server finishes installing k3s before an
+# agent tries to join it. Baking the retry loop into a script keeps that
+# concern out of runcmd, which just calls it once.
 write_files:
   - path: /usr/local/bin/join-k3s.sh
     permissions: '0755'
     content: |
       #!/bin/sh
       # Server may still be booting; retry the join until it answers.
+      # /ping is k3s's own unauthenticated liveness endpoint — this only
+      # confirms the API server is accepting connections yet, it's not a
+      # credential check (K3S_TOKEN below is what actually authorizes the
+      # join once it happens).
       until curl -sk https://${server_ip}:6443/ping >/dev/null 2>&1; do
         echo "waiting for k3s server at ${server_ip}..."
         sleep 5
       done
+      # Agents pass the token via environment variables to the installer
+      # rather than a config.yaml file (contrast server.yaml.tpl) because
+      # get.k3s.io's own install script reads K3S_URL/K3S_TOKEN directly
+      # for the "join an existing cluster" path — there's no separate
+      # agent config file it looks for the way the server has one.
       curl -sfL https://get.k3s.io | \
         K3S_URL="https://${server_ip}:6443" \
         K3S_TOKEN="${k3s_token}" \

terraform/cloud-init/network-config.yaml.tpl

@@ -1,3 +1,13 @@
+# cloud-init's network-config schema (distinct from the #cloud-config
+# user-data schema in server.yaml.tpl/agent.yaml.tpl — no "#cloud-config"
+# header here, and `version: 2` picks the netplan-style dialect). eth0 is
+# the interface name libvirt's virtio NIC presents as inside a fresh
+# Ubuntu cloud image — the only interface that exists at boot, since each
+# node has exactly one network_interface in main.tf's libvirt_domain.
+# `nameservers` points at the gateway rather than a public resolver
+# because that's also where libvirt's own DNS forwarder listens (the `dns
+# { enabled = true }` block on libvirt_network.k3s in main.tf) — it
+# resolves both k3s.local addresses and forwards everything else out.
 version: 2
 ethernets:
   eth0:

terraform/cloud-init/server.yaml.tpl

@@ -14,17 +14,34 @@ package_update: true
 packages:
   - curl
 
+# k3s reads /etc/rancher/k3s/config.yaml automatically on install — this
+# is the file, not command-line flags, specifically so the token never
+# shows up in `ps`/process listings or shell history on the node, and
+# `runcmd` below can stay a one-liner with no secrets in it. 0600 so only
+# root can read it.
 write_files:
   - path: /etc/rancher/k3s/config.yaml
     permissions: '0600'
     content: |
       token: "${k3s_token}"
+      # Every hostname a client might use to reach this server needs to be
+      # in the API server's TLS certificate up front, or that client's TLS
+      # handshake fails outright — it can't be added after the fact
+      # without regenerating the cert. `${hostname}` covers this node's
+      # own name; `${k8s_api_hostname}` is the LAN-only name
+      # docs/04-tofu.md step 5 sets up for kubectl access from elsewhere on
+      # the network — both need to be here even though nothing uses the
+      # second one yet at boot time.
       tls-san:
         - "${hostname}"
         - "${k8s_api_hostname}"
 
 runcmd:
   - curl -sfL https://get.k3s.io | sh -s - server
+  # k3s's own kubeconfig is generated readable only by root (it grants
+  # full cluster-admin access) — this copies it somewhere the unprivileged
+  # `k3s` user can read, so docs/04-tofu.md step 3 can fetch it over SSH
+  # without needing root on the node.
   - mkdir -p /home/k3s/.kube
   - k3s kubectl config view --raw > /home/k3s/.kube/config
   - chown -R k3s:k3s /home/k3s/.kube

terraform/main.tf

@@ -4,12 +4,24 @@ locals {
   prefix_length = split("/", var.network_cidr)[1]
 }
 
+# A dedicated pool, separate from libvirt's own `default` — same
+# "isolated from anything else on the host" motivation as the network
+# below, and it's the fix for the storage-permissions problem
+# docs/02-k3s.md walks through by hand ("Storage: why a raw path in ~
+# doesn't work"): a `dir` pool is a libvirtd-managed location, so volumes
+# created in it come out owned correctly for the `libvirt-qemu` process
+# that actually runs each VM, regardless of what user (`k8s`) asked for it.
 resource "libvirt_pool" "k3s" {
   name = var.storage_pool
   type = "dir"
   path = var.storage_pool_path
 }
 
+# Downloaded once and shared as a read-only backing image for every node's
+# own disk below (libvirt_volume.node's base_volume_id) — the same
+# copy-on-write relationship docs/02-k3s.md explains in detail
+# ("Building the base image..."). Terraform only re-downloads this if
+# base_image_url changes; it doesn't redo it on every apply.
 resource "libvirt_volume" "base" {
   name   = "k3s-base.qcow2"
   pool   = libvirt_pool.k3s.name
@@ -35,6 +47,10 @@ resource "libvirt_network" "k3s" {
   }
 }
 
+# One overlay disk per node, all backed by the single shared base image
+# above — `size` here is the overlay's logical capacity (what the guest
+# OS sees), not how much host disk it actually consumes; qcow2 overlays
+# start small and grow only as the node writes data.
 resource "libvirt_volume" "node" {
   for_each       = var.nodes
   name           = "${each.key}.qcow2"
@@ -44,11 +60,18 @@ resource "libvirt_volume" "node" {
   format         = "qcow2"
 }
 
+# The NoCloud seed ISO for each node — this resource is the Terraform
+# equivalent of `virt-install --cloud-init` in docs/02-k3s.md, just
+# built from two separate templates instead of one hand-assembled file:
 resource "libvirt_cloudinit_disk" "node" {
   for_each = var.nodes
   name     = "${each.key}-cloudinit.iso"
   pool     = libvirt_pool.k3s.name
 
+  # "what should exist on this machine" — which cloud-init template
+  # (server vs agent) depends on each.value.role, so a server node and an
+  # agent node run entirely different runcmd payloads despite sharing this
+  # one resource block. See cloud-init/server.yaml.tpl and agent.yaml.tpl.
   user_data = templatefile("${path.module}/cloud-init/${each.value.role}.yaml.tpl", {
     hostname         = each.key
     ssh_public_key   = var.ssh_public_key
@@ -57,6 +80,11 @@ resource "libvirt_cloudinit_disk" "node" {
     k8s_api_hostname = var.k8s_api_hostname
   })
 
+  # "how should this machine's network be set up" — static, unlike stage
+  # 2's DHCP-based VM (docs/02-k3s.md), because this network
+  # (libvirt_network.k3s below) deliberately runs no DHCP server at all;
+  # every node needs to know its own address up front instead of waiting
+  # on a lease.
   network_config = templatefile("${path.module}/cloud-init/network-config.yaml.tpl", {
     ip            = each.value.ip
     prefix_length = local.prefix_length
@@ -64,6 +92,8 @@ resource "libvirt_cloudinit_disk" "node" {
   })
 }
 
+# The VM itself — everything above this point (pool, base image, per-node
+# overlay, cloudinit ISO, network) exists only to be wired together here.
 resource "libvirt_domain" "node" {
   for_each = var.nodes
   name     = each.key
@@ -73,7 +103,10 @@ resource "libvirt_domain" "node" {
   cloudinit = libvirt_cloudinit_disk.node[each.key].id
 
   network_interface {
-    network_id     = libvirt_network.k3s.id
+    network_id = libvirt_network.k3s.id
+    # false because this network runs no DHCP (see libvirt_network.k3s) —
+    # the node gets its address from cloud-init's network_config instead,
+    # so waiting on a DHCP lease here would just block forever.
     wait_for_lease = false
   }
 

terraform/terraform.tfvars.example

@@ -2,8 +2,8 @@
 # never commit real secrets.
 
 # Default (qemu:///system) assumes you're running tofu on the T630 itself
-# as the 'k8s' user - see docs/SETUP.md step 2. Leave commented out unless
-# you're running Terraform from a separate workstation instead.
+# as the 'k8s' user - see docs/01-bootstrap.md step 2. Leave commented out
+# unless you're running Terraform from a separate workstation instead.
 # libvirt_uri = "qemu+ssh://k8s@t630.lan/system?keyfile=/home/you/.ssh/id_ed25519"
 
 ssh_public_key = "ssh-ed25519 AAAA... you@workstation"

terraform/variables.tf

@@ -1,5 +1,5 @@
 variable "libvirt_uri" {
-  description = "libvirt connection URI. Default assumes tofu/kubectl/flux all run directly on the T630 as the 'k8s' user (see docs/SETUP.md step 2) - simplest option, since node IPs (network_cidr) are only directly reachable from the T630 itself. Use qemu+ssh://k8s@t630.lan/system?keyfile=... instead if you'd rather run Terraform from a separate workstation (you'll then need an SSH tunnel for kubectl/flux to reach node IPs - see step 6)."
+  description = "libvirt connection URI. Default assumes tofu/kubectl/flux all run directly on the T630 as the 'k8s' user (see docs/01-bootstrap.md step 2) - simplest option, since node IPs (network_cidr) are only directly reachable from the T630 itself. Use qemu+ssh://k8s@t630.lan/system?keyfile=... instead if you'd rather run Terraform from a separate workstation (you'll then need an SSH tunnel for kubectl/flux to reach node IPs - see docs/04-tofu.md step 3)."
   type        = string
   default     = "qemu:///system"
 }
@@ -46,7 +46,7 @@ variable "k3s_token" {
 }
 
 variable "k8s_api_hostname" {
-  description = "LAN-only hostname for the k3s API server, added to the server's TLS SAN list so client-cert kubeconfigs validate against it. Resolve it via local DNS only (never a public record) - see docs/SETUP.md step 13."
+  description = "LAN-only hostname for the k3s API server, added to the server's TLS SAN list so client-cert kubeconfigs validate against it. Resolve it via local DNS only (never a public record) - see docs/04-tofu.md step 5."
   type        = string
   default     = "k8s-api.boglabob.com"
 }

terraform/versions.tf

@@ -3,7 +3,16 @@ terraform {
 
   required_providers {
     libvirt = {
-      source  = "dmacvicar/libvirt"
+      source = "dmacvicar/libvirt"
+      # Deliberately pinned to the 0.8.x line, not just "not yet upgraded."
+      # v0.9.0 (Nov 2025) was an intentional, permanent fork of this
+      # provider to a fully-regenerated schema mapping 1:1 to libvirt's own
+      # XML — the maintainer kept 0.8.x alive in parallel specifically for
+      # existing configs like this one, rather than it being a transitional
+      # version to move past. `~> 0.8` (>= 0.8.0, < 0.9.0) stays on that
+      # legacy line on purpose; every resource block in main.tf uses that
+      # schema (verified against the provider's own docs at tag v0.8.3 —
+      # see docs/04-tofu.md's "Where this track actually stands").
       version = "~> 0.8"
     }
   }
@@ -11,7 +20,7 @@ terraform {
 
 # qemu+ssh:// so this can be run from your workstation against the T630;
 # requires the connecting user to be in the T630's `libvirt` and `kvm`
-# groups (see docs/SETUP.md step 2) — no sudo needed after that.
+# groups (see docs/01-bootstrap.md step 2) — no sudo needed after that.
 provider "libvirt" {
   uri = var.libvirt_uri
 }