From 4ea6d8b9e54dd019c6a55d93c873a755b3d3e4b6 Mon Sep 17 00:00:00 2001 From: CodeGit Date: Thu, 3 Sep 2026 19:06:27 +0100 Subject: [PATCH] Updated project to improve guidance --- .forgejo/workflows/terraform.yml | 2 +- .gitignore | 1 + README.md | 109 ++++- apps/headlamp/helmrelease.yaml | 11 +- apps/headlamp/helmrepository.yaml | 3 + apps/headlamp/kustomization.yaml | 2 + apps/headlamp/rbac.yaml | 2 +- apps/hello-app/deployment.yaml | 11 +- apps/hello-app/kustomization.yaml | 2 + apps/podinfo/helmrelease.yaml | 19 + apps/podinfo/helmrepository.yaml | 4 + apps/podinfo/kustomization.yaml | 7 + clusters/homelab/apps.yaml | 7 + docs/01-bootstrap.md | 132 ++++++ docs/{QUICKSTART.md => 02-k3s.md} | 282 ++----------- docs/03-flux.md | 408 +++++++++++++++++++ docs/04-tofu.md | 274 +++++++++++++ docs/Caddyfile.example | 12 +- docs/SETUP.md | 401 ------------------ terraform/cloud-init/agent.yaml.tpl | 14 + terraform/cloud-init/network-config.yaml.tpl | 10 + terraform/cloud-init/server.yaml.tpl | 17 + terraform/main.tf | 35 +- terraform/terraform.tfvars.example | 4 +- terraform/variables.tf | 4 +- terraform/versions.tf | 13 +- 26 files changed, 1091 insertions(+), 695 deletions(-) create mode 100644 docs/01-bootstrap.md rename docs/{QUICKSTART.md => 02-k3s.md} (52%) create mode 100644 docs/03-flux.md create mode 100644 docs/04-tofu.md delete mode 100644 docs/SETUP.md diff --git a/.forgejo/workflows/terraform.yml b/.forgejo/workflows/terraform.yml index 2fc175b..edc8c50 100644 --- a/.forgejo/workflows/terraform.yml +++ b/.forgejo/workflows/terraform.yml @@ -32,4 +32,4 @@ jobs: # No `tofu plan` here: the libvirt provider needs to reach the T630's # libvirt socket, which this ephemeral job container doesn't have # access to. Real applies happen from the T630 itself as the `k8s` - # user (docs/SETUP.md step 5) - fmt/validate is what CI checks. + # user (docs/04-tofu.md step 2) - fmt/validate is what CI checks. diff --git a/.gitignore b/.gitignore index 3acb15e..4bbf4d5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ terraform/terraform.tfvars terraform/*.tfstate terraform/*.tfstate.backup *.env +NOTES-teaching-pass.md diff --git a/README.md b/README.md index 0897d70..e7a23bd 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,21 @@ keeps running its other self-hosted services throughout — this installs as ordinary KVM/libvirt packages next to them, sized deliberately small (3 VMs, 2 vCPU/2GB RAM each), not a hypervisor OS taking over the box. -Two setup paths, sharing everything past the cluster itself: -- **`docs/QUICKSTART.md`** — one manually-created VM (`virt-install`, no - Terraform), fastest way to a real cluster to learn Flux/GitOps on. -- **`docs/SETUP.md`** — the full path, Terraform/OpenTofu provisioning all - 3 VMs. Currently the harder, unfinished track (provider schema issues) — - worth doing properly, but don't block on it. +A four-stage tutorial, each stage building on the last: +- **`docs/01-bootstrap.md`** — one-time host setup: KVM/libvirt, the `k8s` + user, pushing this repo to Forgejo. +- **`docs/02-k3s.md`** — build one VM by hand, get a real k3s cluster + running on it. No Terraform yet. +- **`docs/03-flux.md`** — bootstrap Flux against that cluster, deploy + every app in `apps/` through it (Helm-managed and plain-manifest alike), + wire up the hello-app CI/image-automation loop. +- **`docs/04-tofu.md`** — graduate from the one hand-built VM to a + Terraform/OpenTofu-provisioned 3-node cluster, and point the same Flux + config at it. The harder stage — its HCL is schema-verified against the + pinned provider version but not yet proven with a real `tofu apply` (see + "Where this track actually stands" near the top of that doc) — worth + finishing properly, but don't block on it; stages 2-3 already get you a + complete, working GitOps loop without it. ## The loop @@ -33,16 +42,70 @@ Flux (running in the cluster) -> applies clusters/homelab/** and apps/** to k3s Caddy (Podman, existing) is the public front door for boglabob.com, but only -for the low-stakes apps — the Dashboard and the k3s API server stay off it: +for the low-stakes apps — Headlamp and the k3s API server stay off it: git.boglabob.com -> Forgejo (existing) podinfo.boglabob.com -> Traefik (k3s ingress) -> podinfo hello.boglabob.com -> Traefik (k3s ingress) -> hello-app - [Dashboard] -> kubectl port-forward only, never a public hostname + [Headlamp] -> kubectl port-forward only, never a public hostname [k3s API, port 6443] -> reachable only via the T630 (SSH tunnel or run kubectl there directly), no proxy, no public port - see docs/Caddyfile.example and docs/SETUP.md steps 12-13 + see docs/Caddyfile.example, docs/03-flux.md step 5 (Headlamp), and + docs/04-tofu.md step 5 (LAN/remote kubectl access) ``` +## Why it's structured this way + +Each layer below only knows about the one directly beneath it — that's +deliberate, not incidental, and it's what makes the pieces independently +replaceable: + +- **Terraform provisions VMs, nothing more.** It doesn't install k3s itself + or configure Flux. Its job ends at "three VMs exist, on this network, with + this cloud-init data attached." That's what makes `docs/02-k3s.md` + possible at all — a single `virt-install` VM can stand in for the whole + Terraform layer, because everything above it only cares that *some* VM + with a `k3s` user and a working kubeconfig exists, not how it got there — + which is exactly why Terraform is stage 4, not stage 1: everything above + it gets learned and proven out on one hand-built VM first. +- **cloud-init installs k3s, nothing about Flux.** It runs `get.k3s.io` and + stages a kubeconfig — full stop. Flux isn't baked into the image or the + cloud-init data because `flux bootstrap` needs to *commit into this repo* + (the generated `clusters/homelab/flux-system/` manifests), which only + makes sense as a deliberate, one-time, human-run command against a + cluster that already exists and can reach Forgejo — not something to + automate blindly on every VM boot. +- **`flux bootstrap` is a one-time bridge, not a recurring step.** It's the + only command in this whole project that goes API-server-side *and* + repo-side in one shot: it installs the controllers, and it writes + `clusters/homelab/flux-system/` back into git. After that one run, the + cluster is entirely git-driven — rerunning `flux bootstrap` against the + same repo is idempotent (safe if you need to recover a cluster, or to + point the same config at a new cluster — see `docs/04-tofu.md` step 4), + but nothing after that first run ever needs a human to run + `kubectl apply` again. +- **`clusters/homelab/apps.yaml` fans out on purpose, instead of one giant + Kustomization pointing at `apps/`.** Each app gets its own Flux + `Kustomization` object (see `clusters/homelab/apps.yaml`) with its own + `interval` and its own `prune: true`. That means a bad manifest in + `apps/hello-app` can't block `apps/podinfo` from reconciling — each app's + sync loop is independent, and you can watch/debug them individually with + `flux get kustomization ` instead of one opaque blob. +- **Helm apps (podinfo, headlamp) vs. plain manifests (hello-app).** + podinfo and headlamp are third-party charts — Flux's `HelmRepository` + + `HelmRelease` pair is what lets Flux track and upgrade an upstream chart + without this repo vendoring it. `hello-app` is *this* project's own code, + so it's plain Kubernetes YAML (`Deployment`/`Service`/`Ingress`) — there's + no chart to track, and plain manifests are simpler to diff/read for + something this small. +- **Image automation exists only for `hello-app`, not podinfo/headlamp.** + `ImageRepository`/`ImagePolicy`/`ImageUpdateAutomation` + (`apps/hello-app/image-automation.yaml`) close the loop from "CI pushed a + new image tag" to "the cluster is running it" — that's only relevant for + an app whose image *this repo's own CI* builds. podinfo/headlamp pin a + chart version range instead (`>=6.0.0`, `>=0.40.0`) and get upgraded by + bumping that constraint by hand, since there's no CI producing new tags + for them to watch. + ## Directory layout - `terraform/` — OpenTofu config that provisions the k3s VMs via KVM/libvirt @@ -51,7 +114,7 @@ for the low-stakes apps — the Dashboard and the k3s API server stay off it: - `terraform/cloud-init/` — cloud-init templates that install k3s server/agent on first boot. - `clusters/homelab/` — Flux's own config for this cluster (populated by - `flux bootstrap`, see docs/SETUP.md step 7). + `flux bootstrap`, see docs/03-flux.md step 2). - `apps/podinfo/` — first GitOps app: a HelmRelease for the standard Flux demo app (podinfo). No CI needed — good for validating the Flux sync loop works. - `apps/hello-app/` — capstone app, and the public showcase piece at @@ -61,28 +124,32 @@ for the low-stakes apps — the Dashboard and the k3s API server stay off it: automatically. The page itself shows the live commit SHA and deploy timestamp injected by CI, as proof the pipeline is really running rather than a static screenshot. -- `apps/kubernetes-dashboard/` — the official Kubernetes Dashboard, with a - cluster-admin token (`apps/kubernetes-dashboard/rbac.yaml`). No ingress — - access is via `kubectl port-forward` only (see docs/SETUP.md step 12). +- `apps/headlamp/` — a Kubernetes UI (replacing the now-unmaintained + Kubernetes Dashboard), with a cluster-admin token + (`apps/headlamp/rbac.yaml`). No ingress — access is via `kubectl + port-forward` only (see docs/03-flux.md step 5). - `.forgejo/workflows/` — CI: terraform validate/plan on PRs, build+push hello-app image on merge to main, via kaniko under a rootless Podman - runner (no docker.sock, no sudo — see docs/SETUP.md step 9). + runner (no docker.sock, no sudo — see docs/03-flux.md step 6). - `docs/Caddyfile.example` — the reverse-proxy blocks for podinfo/hello-app to add to your existing Caddy (Podman) config. -## Prerequisites (see docs/SETUP.md for the full walkthrough) +## Prerequisites (see docs/01-bootstrap.md onward for the full walkthrough) - `qemu-kvm`/`libvirt` installed on the T630 (alongside its existing services) - Forgejo instance reachable, with this repo pushed to it and Actions enabled -- `tofu`, `kubectl`, `flux` CLI — run on the T630 itself as the unprivileged +- `kubectl`, `flux`, `tofu` CLI — run on the T630 itself as the unprivileged `k8s` user (simplest, since the VMs' network is only reachable from there) ## Milestones -1. KVM/libvirt installed on the T630, `k8s` user created -2. `terraform apply` — 3 VMs come up, k3s cluster forms -3. `flux bootstrap` against this Forgejo repo -4. Push `apps/podinfo` — watch Flux deploy it with no manual `kubectl apply` -5. Wire `.forgejo/workflows/build-hello-app.yml` — push a code change to +1. KVM/libvirt installed on the T630, `k8s` user created (`docs/01-bootstrap.md`) +2. One hand-built VM, k3s running on it, kubectl talking to it (`docs/02-k3s.md`) +3. `flux bootstrap` against this Forgejo repo — podinfo/headlamp/hello-app + deploy with no manual `kubectl apply` (`docs/03-flux.md`) +4. Wire `.forgejo/workflows/build-hello-app.yml` — push a code change to `apps/hello-app/src`, watch CI build → registry → Flux auto-deploy + (`docs/03-flux.md` step 7) +5. `tofu apply` — 3 VMs come up, k3s cluster forms, same Flux config + re-bootstrapped against it (`docs/04-tofu.md`) 6. Stretch: sealed-secrets/SOPS, kube-prometheus-stack, cert-manager diff --git a/apps/headlamp/helmrelease.yaml b/apps/headlamp/helmrelease.yaml index ac8614b..cd72527 100644 --- a/apps/headlamp/helmrelease.yaml +++ b/apps/headlamp/helmrelease.yaml @@ -1,8 +1,9 @@ -# Deliberately no ingress here — the official Dashboard grants whatever the -# logged-in identity can do, and a leaked cluster-admin token to a publicly -# reachable Dashboard is a well-known real-world breach vector (see e.g. the -# 2018 Tesla incident). Access is via `kubectl port-forward` only -# (docs/SETUP.md step 12) — no standing hostname, no attack surface between +# Deliberately no ingress here — Headlamp (like the Kubernetes Dashboard it +# replaces, and like any cluster admin UI) grants whatever the logged-in +# identity can do, and a leaked cluster-admin token to a publicly reachable +# admin UI is a well-known real-world breach vector (see e.g. the 2018 +# Tesla incident). Access is via `kubectl port-forward` only +# (docs/03-flux.md step 5) — no standing hostname, no attack surface between # sessions. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease diff --git a/apps/headlamp/helmrepository.yaml b/apps/headlamp/helmrepository.yaml index ac5d33d..7cce822 100644 --- a/apps/headlamp/helmrepository.yaml +++ b/apps/headlamp/helmrepository.yaml @@ -1,3 +1,6 @@ +# Same role as apps/podinfo/helmrepository.yaml: just the chart index +# source-controller polls, cached as an artifact for helmrelease.yaml to +# install from. apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: diff --git a/apps/headlamp/kustomization.yaml b/apps/headlamp/kustomization.yaml index 65695e5..6e36b9c 100644 --- a/apps/headlamp/kustomization.yaml +++ b/apps/headlamp/kustomization.yaml @@ -1,3 +1,5 @@ +# Inventory of this app's manifests — see apps/podinfo/kustomization.yaml +# for why listed order here doesn't matter for apply ordering. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/apps/headlamp/rbac.yaml b/apps/headlamp/rbac.yaml index 07f8f0f..a3d6b3b 100644 --- a/apps/headlamp/rbac.yaml +++ b/apps/headlamp/rbac.yaml @@ -1,5 +1,5 @@ # Cluster-admin ServiceAccount + long-lived token, used both to log in to -# the Dashboard (docs/SETUP.md step 12) and as the client identity for +# Headlamp (docs/03-flux.md step 5) and as the client identity for # direct LAN kubectl access if you'd rather not manage the SSH-fetched # kubeconfig from step 6. Fine for a single-user homelab; split into # narrower-scoped accounts if more people get access later. diff --git a/apps/hello-app/deployment.yaml b/apps/hello-app/deployment.yaml index ff0ef3c..4562f25 100644 --- a/apps/hello-app/deployment.yaml +++ b/apps/hello-app/deployment.yaml @@ -1,3 +1,7 @@ +# This app's own Deployment/Service/Ingress, hand-written rather than a +# Helm chart (see README.md "Why it's structured this way") — there's no +# upstream chart to track since the image is built by this repo's own CI +# (.forgejo/workflows/build-hello-app.yml). apiVersion: apps/v1 kind: Deployment metadata: @@ -14,11 +18,16 @@ spec: app: hello-app spec: # Uncomment if the hello-app package is set to private in Forgejo - # (see docs/SETUP.md step 9). + # (see docs/03-flux.md step 6). # imagePullSecrets: # - name: forgejo-registry containers: - name: hello-app + # Not a comment — this exact-format annotation is how Flux's + # image-automation "Setters" strategy (image-automation.yaml's + # ImageUpdateAutomation) finds this line and knows to rewrite the + # tag below in place, rather than needing this field templated. + # "hello-app:hello-app" is ":". # {"$imagepolicy": "hello-app:hello-app"} image: git.boglabob.com/codegit/hello-app:latest ports: diff --git a/apps/hello-app/kustomization.yaml b/apps/hello-app/kustomization.yaml index 67ce325..a80c6c7 100644 --- a/apps/hello-app/kustomization.yaml +++ b/apps/hello-app/kustomization.yaml @@ -1,3 +1,5 @@ +# Inventory of this app's manifests — see apps/podinfo/kustomization.yaml +# for why listed order here doesn't matter for apply ordering. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/apps/podinfo/helmrelease.yaml b/apps/podinfo/helmrelease.yaml index fb39eda..92a7a0a 100644 --- a/apps/podinfo/helmrelease.yaml +++ b/apps/podinfo/helmrelease.yaml @@ -1,13 +1,27 @@ +# First app in this repo, deliberately picked because it needs nothing +# this project builds itself — no CI, no custom image, just a public chart +# — so a green `flux get kustomization podinfo` here proves the whole +# source-controller -> helm-controller -> cluster loop works before +# hello-app adds a CI dependency on top of it. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: name: podinfo namespace: podinfo spec: + # How often helm-controller re-checks this HelmRelease against the chart + # in helmrepository.yaml's cache and re-reconciles if they've drifted — + # independent of how often source-controller refreshes that cache (1h, + # see helmrepository.yaml). interval: 10m chart: spec: chart: podinfo + # A range, not a pin: helm-controller installs the latest version + # matching this constraint and re-installs on every new matching + # release the upstream chart repo publishes. Bump the lower bound by + # hand to move the floor forward; there's no CI here to do it + # automatically (contrast apps/hello-app's image automation). version: ">=6.0.0" sourceRef: kind: HelmRepository @@ -17,9 +31,14 @@ spec: replicaCount: 1 ingress: enabled: true + # k3s ships Traefik as its built-in ingress controller — this is + # its IngressClass name, not a choice made by this chart. className: traefik hosts: - host: podinfo.boglabob.com paths: - path: / + # Lets the chart's own Ingress template pick the matching + # type for its backend, rather than this repo asserting one + # (podinfo's chart uses this instead of a plain "Prefix"). pathType: ImplementationSpecific diff --git a/apps/podinfo/helmrepository.yaml b/apps/podinfo/helmrepository.yaml index 18dfce5..feaa535 100644 --- a/apps/podinfo/helmrepository.yaml +++ b/apps/podinfo/helmrepository.yaml @@ -1,3 +1,7 @@ +# Declares where podinfo's chart index lives. On its own this does nothing +# to the cluster — source-controller just fetches this URL every `interval` +# and caches the resulting chart index as an artifact. helmrelease.yaml is +# what actually asks for the chart to be installed. apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: diff --git a/apps/podinfo/kustomization.yaml b/apps/podinfo/kustomization.yaml index b4a3d7c..921112d 100644 --- a/apps/podinfo/kustomization.yaml +++ b/apps/podinfo/kustomization.yaml @@ -1,3 +1,10 @@ +# Plain kustomize, no patches/overlays — this just lists which files +# belong to this app so `kustomize build apps/podinfo` (or Flux's own +# `Kustomization` object of the same name in clusters/homelab/apps.yaml, +# pointed at this directory) has one thing to apply. Order matters only in +# that namespace.yaml has to exist before objects that reference it, but +# kubectl/kustomize sort by kind anyway, so this is really just an +# inventory. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/clusters/homelab/apps.yaml b/clusters/homelab/apps.yaml index 2bada30..d5eabc1 100644 --- a/clusters/homelab/apps.yaml +++ b/clusters/homelab/apps.yaml @@ -1,6 +1,13 @@ # flux bootstrap points the flux-system Kustomization at ./clusters/homelab # and applies everything under it (prune: true), so these Kustomization # objects are how the rest of the repo gets pulled in. +# +# One object per app, each with its own sourceRef/interval/prune, rather +# than a single Kustomization pointed at all of apps/ — see README.md +# "Why it's structured this way" for the reasoning (independent failure +# domains, independent `flux get kustomization ` debugging). All +# three share the same sourceRef because they all come from this one repo +# — sourceRef is what to fetch from, path is where inside it to look. apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/docs/01-bootstrap.md b/docs/01-bootstrap.md new file mode 100644 index 0000000..4f16bec --- /dev/null +++ b/docs/01-bootstrap.md @@ -0,0 +1,132 @@ +# Stage 1: Bootstrap + +This project is a four-stage tutorial, each stage building on the last: + +1. **`docs/01-bootstrap.md`** (this doc) — one-time host setup: KVM/libvirt, + an unprivileged user to run everything as, and pushing this repo to + Forgejo. Nothing here is specific to k3s, Flux, or Terraform — every + later stage depends on it. +2. **`docs/02-k3s.md`** — build one VM by hand and get a real k3s cluster + running on it. No Flux yet, no Terraform — just a working cluster and a + kubeconfig that talks to it. +3. **`docs/03-flux.md`** — bootstrap Flux against that cluster and deploy + this repo's apps through it, including what Helm chart management looks + like under GitOps (podinfo, headlamp) versus plain manifests + (hello-app), and the CI loop that builds and auto-deploys hello-app. +4. **`docs/04-tofu.md`** — graduate from the one hand-built VM to a proper + 3-node cluster provisioned by Terraform/OpenTofu, and point the same + Flux config at it. + +Each stage says exactly which earlier steps it actually depends on, rather +than assuming you need everything done up front — stage 2, for instance, +only needs steps 1-2 below. + +Assumes: the T630 is an existing Debian box already running other +self-hosted services — this project installs alongside those as ordinary +packages (`qemu-kvm`/`libvirt`), not a hypervisor OS replacing Debian. +Forgejo is already running and reachable at `https://git.boglabob.com`, and +you can point DNS records under `boglabob.com` at hosts on your network +(directly, or via whatever reverse proxy/tunnel already gets +`git.boglabob.com` there). + +--- + +## 1. Install KVM/libvirt on the T630 + +Ordinary packages, no reboot into an installer, nothing else on the box is +touched: + +```sh +# on the T630 +sudo apt update +sudo apt install -y qemu-kvm libvirt-daemon-system libvirt-clients virtinst + +# confirm hardware virtualization is available (T630's Xeons support it) +sudo kvm-ok +``` + +## 2. Create the unprivileged 'k8s' user + +One dedicated, no-sudo user for everything this project touches: driving +`virsh`/`kubectl`/`flux`/`tofu` against libvirt, and running the Forgejo +Actions runner later (`docs/03-flux.md` step 6). It needs group membership +to talk to libvirt — that's a one-time root action; nothing it does +afterwards needs `sudo`. + +```sh +sudo useradd -m -s /bin/bash k8s # one-time, needs root to create the user itself +sudo usermod -aG libvirt,kvm k8s +sudo loginctl enable-linger k8s # lets its services keep running after logout + +# as k8s, from here on (sudo -iu, not su -, since k8s has no password set): +sudo -iu k8s +ssh-keygen -t ed25519 -C "k3s-homelab" -f ~/.ssh/id_ed25519 # only needed if you'll SSH in as k8s day-to-day +virsh -c qemu:///system list --all # sanity check: should run with no permission error, no sudo +``` + +Do the rest of this project logged in as `k8s` on the T630 itself (`ssh +k8s@t630`) — VM IPs live on a private libvirt network that's only directly +reachable from the T630, so this is the simplest place to run +`kubectl`/`flux`/`tofu` from. (If you'd rather drive Terraform from your +own workstation instead once you reach stage 4, see the `libvirt_uri` +comment in `terraform/terraform.tfvars.example` — you'll then need an SSH +tunnel for kubectl/flux to reach node IPs.) + +## 3. Generate the secret k3s needs + +```sh +openssl rand -hex 32 # -> k3s_token +``` + +This is the shared token agent nodes use to join a k3s server — irrelevant +for stage 2's single-node VM (a lone server needs no one to join it), but +generate it now while you're doing one-time setup; stage 4's +Terraform-provisioned multi-node cluster is what actually uses it. + +## 4. Push this repo to Forgejo + +Doing this before any cluster exists (rather than after) means `k8s` can +get the repo with a plain `git clone` later, instead of needing a one-off +copy handed to it — and any future change to this repo just needs a +`git pull` on the T630. Repo/owner used throughout this project: +`codegit/cloud-demo` (already baked into `apps/hello-app/deployment.yaml` +and `image-automation.yaml`'s image references — no placeholder-swapping +needed). + +1. On Forgejo (`https://git.boglabob.com`), as `codegit`: **+ → New + Repository** → name `cloud-demo`. Leave it empty — don't initialize with + a README/`.gitignore`/license, since this repo already has its own. + Visibility (public/private) is your call; either works, since access for + `k8s`/Flux/CI goes through the tokens below regardless. +2. Locally, wherever you're editing this repo (`maq`): + ```sh + git init # if not already + git add . + git commit -m "initial scaffold" + git remote add origin https://git.boglabob.com/codegit/cloud-demo.git + git push -u origin main + ``` +3. Generate two access tokens (`Settings → Applications → Generate New + Token`), scoped as narrowly as Forgejo's token UI allows to repository + read/write: + - **`k8s-readonly`** — read-only. Used only for `k8s`'s own manual + `git clone`/`pull` on the T630 — never leaves that box, isn't used by + anything automated. Not needed until `docs/04-tofu.md` (that's the + first stage that clones this repo onto the T630 rather than editing + it from your workstation). + - **`flux-write`** — read/write. Used once, as a `flux bootstrap` + argument (`docs/03-flux.md` step 2); Flux stores it as a Kubernetes + Secret inside the cluster from then on (`ImageUpdateAutomation`'s + commits back, in the hello-app section of that stage, reuse that same + in-cluster Secret) — it's never written to `k8s`'s filesystem at all. + + Using HTTPS tokens instead of `k8s`'s SSH key (`~/.ssh/id_ed25519`, from + step 2) sidesteps an open question: Forgejo's git-SSH port isn't + reachable from this desktop through your router (see the SSH + troubleshooting earlier in this project's history), and whether it's + reachable from `k8s` on the T630 itself was never actually confirmed + either. HTTPS (443, via Caddy) is already proven to work, so both + tokens use that instead. Copy both token values now — Forgejo only + shows them once. + +Next: `docs/02-k3s.md` — you only need steps 1-2 above to start it. diff --git a/docs/QUICKSTART.md b/docs/02-k3s.md similarity index 52% rename from docs/QUICKSTART.md rename to docs/02-k3s.md index 3292f7f..2a7d627 100644 --- a/docs/QUICKSTART.md +++ b/docs/02-k3s.md @@ -1,24 +1,17 @@ -# Quickstart: manual cluster (no Terraform) +# Stage 2: A real k3s cluster, built by hand -A real k3s cluster, built by hand, so Flux/GitOps can be learned right away -instead of waiting on the Terraform/libvirt provider work in `docs/SETUP.md` -to get sorted (that provider did a breaking rewrite between 0.8.x and -0.9.x, and the HCL needs writing against the real schema, not memory). +Stage 1 (`docs/01-bootstrap.md`) got KVM/libvirt installed and the `k8s` +user created — that's all this stage needs (its steps 1-2; skip 3-4 for +now, they're not needed until stage 3). This stage builds one throwaway VM +directly with `virt-install` — no Terraform involved at all — and installs +k3s on it, so stage 3 has a real cluster to bootstrap Flux/GitOps against. +Terraform doesn't show up until stage 4, once you already understand what +it's automating. -This is one throwaway VM, created directly with `virt-install` — no -Terraform involved at all. Everything here explains the *why*, not just -the *what*: read each section before running its commands, and where a -config file is being built (the cloud-init data, the Caddy block), treat -the snippets as pieces to assemble into your own file, not something to -paste wholesale. - -Shares steps 1–4 of `docs/SETUP.md` as prerequisites: -- Step 1: KVM/libvirt packages installed on the T630. -- Step 2: the unprivileged `k8s` user exists, in the `libvirt`/`kvm` groups, - with an SSH keypair at `~/.ssh/id_ed25519`. -- Step 3: skip — no join token needed here (see "one node is enough" below). -- Step 4: `cloud-demo` pushed to Forgejo, with a `k8s-readonly` and a - `flux-write` token generated. +Everything here explains the *why*, not just the *what*: read each section +before running its commands, and where a config file is being built (the +cloud-init data, the Caddy block later on), treat the snippets as pieces to +assemble into your own file, not something to paste wholesale. Everything below runs as `k8s` on the T630 (`sudo -iu k8s`). @@ -30,10 +23,10 @@ A k3s **server** node runs the control plane (API server, scheduler, etcd/SQLite) *and* schedules ordinary workloads onto itself unless you explicitly disable that. So a single server, with no agents, is already a complete, working cluster — nothing here needs a join token or a second -VM. The 3-node design in `docs/SETUP.md` exists to mirror a more +VM. The 3-node design in `docs/04-tofu.md` exists to mirror a more realistic multi-node cluster once Terraform can build it repeatedly and -disposably; for learning Flux and GitOps, that extra shape doesn't buy -you anything yet. +disposably; for learning k3s, Helm, and Flux/GitOps, that extra shape +doesn't buy you anything yet. ## Two libvirt connections, and why it matters @@ -43,8 +36,8 @@ perspective there are (at least) two separate connections: - `qemu:///system` — the shared, host-wide instance. VMs here can use privileged networking (bridges, NAT with DHCP), and management access is gated by group membership (`libvirt`/`kvm`) checked via polkit — which is - exactly what step 2's `usermod -aG libvirt,kvm k8s` set up. No sudo - needed for any command below; that group membership *is* the + exactly what bootstrap step 2's `usermod -aG libvirt,kvm k8s` set up. No + sudo needed for any command below; that group membership *is* the authorization. - `qemu:///session` — a private, per-user instance with no special privileges, and critically, its own separate storage pools and networks @@ -76,6 +69,8 @@ the simplest case). The key property: creating or writing a volume inside a pool goes through libvirtd's API, not through `k8s`'s own filesystem permissions — so it's libvirtd (already running with the right privileges) that handles ownership correctly, regardless of what user asked for it. +(Stage 4's Terraform config hits this exact same problem and solves it the +same way — see `terraform/main.tf`'s `libvirt_pool` resource.) Check what pools/networks already exist: @@ -134,7 +129,10 @@ VM built from the same base. Instead, create a **copy-on-write overlay**: a second volume that starts out empty and only stores the *differences* from its backing volume. Reads that haven't been changed transparently fall through to the base image; writes go into the overlay. This is the -same relationship a Docker image layer has to its base layer. +same relationship a Docker image layer has to its base layer. (Same +relationship as `terraform/main.tf`'s `libvirt_volume.base` / +`libvirt_volume.node` pair in stage 4 — one shared base, one overlay per +node.) ```sh virsh -c qemu:///system vol-create-as default k3s-manual.qcow2 20G --format qcow2 \ @@ -152,15 +150,16 @@ first boot, driven by data supplied externally rather than baked into the image. `virt-install`'s `--cloud-init` flag builds a small ISO (the "NoCloud" datasource) containing that data and attaches it to the VM; cloud-init, already installed in the image, detects it automatically at -boot. +boot. (`terraform/main.tf`'s `libvirt_cloudinit_disk` resource in stage 4 +is the same mechanism, just built by Terraform instead of by hand.) Two separate pieces of data go in, and they answer different questions: - **`user-data`** — *what should exist on this machine*: users, packages, commands to run. Written as `#cloud-config` YAML. - **`network-config`** — *how should this machine's network be set up*. - This quickstart skips it entirely and lets the VM get an address via - DHCP from the `default` network instead (the Terraform track, by + This stage skips it entirely and lets the VM get an address via DHCP + from the `default` network instead (stage 4's Terraform config, by contrast, uses this for static IPs, since it manages its own isolated network). @@ -340,227 +339,6 @@ export KUBECONFIG=~/.kube/config-manual kubectl get nodes # expect 1 Ready node ``` -## Flux: what's actually happening, mechanism by mechanism - -"Flux" isn't one program watching your repo by magic — it's a handful of -ordinary Kubernetes controllers (just Deployments, like anything else -you'd run on the cluster), each one understanding a couple of **Custom -Resource Definitions** — CRDs extend the Kubernetes API with new object -kinds, the same way `Deployment` or `Service` are built-in kinds. -`GitRepository` and `Kustomization` are two such kinds Flux adds. Nothing -about any of this is special-cased outside the normal Kubernetes API — -it's the exact same "define an object, a controller notices it and acts" -loop that runs the whole rest of Kubernetes. - -Install the CLI the same no-sudo, direct-binary-release way as `kubectl`: - -```sh -FLUX_VERSION=$(curl -s https://api.github.com/repos/fluxcd/flux2/releases/latest | grep tag_name | cut -d '"' -f4 | sed 's/^v//') -curl -L -o /tmp/flux.tar.gz "https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" -tar -xzf /tmp/flux.tar.gz -C ~/.local/bin flux -rm /tmp/flux.tar.gz -flux --version -``` - -Forgejo isn't a Flux-native provider the way GitHub/GitLab are, so this -uses the generic git bootstrap — over HTTPS with the `flux-write` token -from step 4, not SSH (Forgejo's git-SSH port turned out not to be reliably -reachable from either this desktop or the T630 — see the earlier -troubleshooting in this project's history): - -```sh -flux check --pre --kubeconfig ~/.kube/config-manual - -flux bootstrap git \ - --url=https://git.boglabob.com/codegit/cloud-demo \ - --branch=main \ - --path=clusters/homelab \ - --username=codegit \ - --password= \ - --token-auth \ - --kubeconfig ~/.kube/config-manual -``` - -That one command did five genuinely separate things. Go look at each — -these commands work against the cluster you already bootstrapped: - -**1. It installed the controllers** — plain Kubernetes Deployments, no -different in kind from anything else running on the cluster: - -```sh -kubectl -n flux-system get deployments -``` - -`source-controller`, `kustomize-controller`, `helm-controller`, -`notification-controller`. Alongside them, it registered the CRDs those -controllers understand: - -```sh -kubectl get crds | grep fluxcd -``` - -**2. It created a `GitRepository` object** — this is the whole "what repo -am I watching" declaration, and nothing more. `source-controller`'s job is -entirely mechanical: every `interval` (default 1m), do a real `git fetch` -against `.spec.url`/`.spec.ref`; if the commit SHA changed, package that -tree into a `.tar.gz`, and record its location in `.status.artifact`. -That's the entire job — it doesn't know or care what's *in* the repo. - -```sh -kubectl -n flux-system get gitrepository flux-system -o yaml -``` - -Look at `.status.conditions` and `.status.artifact` — that's the result of -an actual git fetch that already happened, not a static config. - -**3. It created a `Kustomization` object** (confusingly, the same name as -the `kustomization.yaml` files already sitting in `apps/podinfo/` etc. — -related but not identical). `kustomize-controller`'s job, on its own -interval: fetch the artifact `source-controller` produced, run the real -`kustomize` tool against `.spec.path` inside it (the exact same tool a -plain `kustomize build apps/podinfo` would run locally against those -`kustomization.yaml` files), and apply the resulting objects via the -Kubernetes API — the automated equivalent of you running -`kubectl apply -f <(kustomize build apps/podinfo)` yourself, on a timer, -forever. `prune: true` (set on all the `Kustomization` objects in this -repo) means it also *deletes* anything it previously created that's no -longer present in the current git state — that's what makes it -self-healing rather than just "apply once." - -```sh -kubectl -n flux-system get kustomization -``` - -**This is also why `apps/podinfo`, `apps/hello-app`, and -`apps/kubernetes-dashboard` started deploying without you ever running -`kubectl apply` on them.** Bootstrap's own `Kustomization` watches -`clusters/homelab` with `prune: true`. `clusters/homelab/apps.yaml` -(already sitting in the repo, hand-written earlier in this project) itself -just *defines more `Kustomization` objects*, one per app — so the first -one picks it up as part of its own normal reconcile, creates those three -child `Kustomization`s, and each of *those* then does its own -fetch-and-apply against its own app directory. It's the same mechanism -recursing, not a special case. - -**4. It stored your credential as a Kubernetes Secret** — `--password` -here is the `flux-write` token, not an account password. Bootstrap -base64-wraps it into a Secret that matches what the `GitRepository`'s -`.spec.secretRef` points at: - -```sh -kubectl -n flux-system get secret flux-system -o yaml -``` - -The `password` field is base64 (`| base64 -d` to read it) — this is -exactly the same credential `git clone https://user:token@host/repo` would -use, just read by `source-controller` on every fetch instead of typed by -you once. It's never written to `k8s`'s own filesystem. - -**5. It committed that config back into the repo itself** — `git pull` in -your desktop clone and look at `clusters/homelab/flux-system/`. The -`GitRepository`/`Kustomization` objects you just inspected live in the -cluster *because* those exact YAML files are committed there — so -rebuilding this cluster from scratch would mean running `flux bootstrap` -again (or even just `kubectl apply -f clusters/homelab/flux-system/`) and -landing in the identical state. The fact that Flux watches this repo is -itself declared *in* this repo. - -If you want to watch a reconcile happen live rather than just inspect the -end state: - -```sh -kubectl -n flux-system logs deploy/source-controller -f -``` - -then in another terminal, make any commit and push it — you'll see the -next fetch pick it up within the interval. - -## Verifying the GitOps loop actually works - -```sh -flux get kustomizations --watch -``` - -Watch until `podinfo`, `hello-app`, and `kubernetes-dashboard` all show -`Ready: True`, then confirm pods actually landed: - -```sh -kubectl -n podinfo get pods -kubectl -n hello-app get pods -kubectl -n kubernetes-dashboard get pods -``` - -If that's all healthy, the entire chain — Forgejo repo → Flux → this -cluster — is working end to end with nothing manually `kubectl apply`'d. - -## Exposing podinfo/hello-app through Caddy - -k3s's bundled ingress controller (Traefik) is already listening on this -node's own IP, port 80, routing by the `Host:` header from each app's -`Ingress` resource (already defined in `apps/podinfo` and -`apps/hello-app`). Caddy just needs to forward matching requests there. - -Two DNS records, as CNAMEs pointed at `git.boglabob.com` rather than -duplicating its IP directly — CNAME means "this name is an alias for that -one," so there's one place (that record) to update if the underlying IP -ever changes, instead of several: - -``` -podinfo.boglabob.com CNAME git.boglabob.com -hello.boglabob.com CNAME git.boglabob.com -``` - -Then in Caddy's own config, using the VM's pinned IP from earlier: - -``` -podinfo.boglabob.com { - reverse_proxy http://:80 -} - -hello.boglabob.com { - reverse_proxy http://:80 -} -``` - -```sh -podman exec caddy reload --config /etc/caddy/Caddyfile -``` - -The Dashboard and the k3s API server are deliberately **not** here — -`docs/SETUP.md` steps 12–13 cover why (both are cluster-admin-capable, and -exposing either publicly is the exact pattern behind real breaches like -Tesla's 2018 incident) and how to reach them instead (`kubectl -port-forward`, and LAN/tunnel-only kubectl access). Both apply to this -cluster exactly as written there. - -## Registering the Forgejo Actions runner - -`docs/SETUP.md` step 9 applies as written — it's about the `k8s` user and -rootless Podman, not about which cluster exists. Worth understanding -before running it: the runner would normally get root-equivalent power -over its host via a mounted `docker.sock`; instead it runs as `k8s` itself -using rootless Podman's own API socket, and `build-hello-app.yml` builds -images with `kaniko` (no daemon, no elevated privileges needed at all) — -so nothing in that pipeline ever touches `sudo`. - -## Tearing this down - -Once the Terraform track is ready and `tofu apply` brings up the real -3-node cluster, remove this one: - -```sh -virsh -c qemu:///system destroy k3s-manual # stop it -virsh -c qemu:///system undefine k3s-manual --remove-all-storage # VM + overlay disk -virsh -c qemu:///system vol-delete --pool default k3s-manual-base.qcow2 # base image isn't - # attached to the VM - # directly, needs its - # own delete -rm ~/.kube/config-manual -rm -rf ~/vms -``` - -Nothing else needs cleaning up — Flux's own state lived entirely inside -that VM's cluster and goes away with it. The Forgejo repo, both tokens, -and the Forgejo Actions runner registration are all cluster-independent -and carry over unchanged; point `flux bootstrap` at the new cluster's -kubeconfig once it exists. +A real, working single-node k3s cluster. Next: `docs/03-flux.md` — you'll +need bootstrap steps 3-4 (the `k3s_token` secret isn't used until stage 4, +but the Forgejo tokens are needed starting now) before continuing. diff --git a/docs/03-flux.md b/docs/03-flux.md new file mode 100644 index 0000000..679f215 --- /dev/null +++ b/docs/03-flux.md @@ -0,0 +1,408 @@ +# Stage 3: Flux, Helm, and this repo's apps + +By the end of stage 2 (`docs/02-k3s.md`) you have a working k3s cluster and +a kubeconfig at `~/.kube/config-manual`. This stage also needs bootstrap +steps 3-4 from `docs/01-bootstrap.md` — the Forgejo repo pushed, and the +`flux-write` token generated (the `k3s_token` secret still isn't used until +stage 4). + +This stage bootstraps Flux against that cluster, and through it deploys +every app in `apps/`: two managed via Helm (podinfo, headlamp — this is +also where Helm itself gets explained, since Flux is the only way this +project ever touches Helm) and one via plain manifests plus a CI pipeline +(hello-app). + +Everything below runs as `k8s` on the T630, with +`export KUBECONFIG=~/.kube/config-manual` already set from stage 2. + +--- + +## 1. Install the Flux CLI + +Same no-sudo, direct-binary-release pattern as `kubectl` in stage 2 +(`brew` assumes Homebrew, which isn't a given on a bare Debian box): + +```sh +FLUX_VERSION=$(curl -s https://api.github.com/repos/fluxcd/flux2/releases/latest | grep tag_name | cut -d '"' -f4 | sed 's/^v//') +curl -L -o /tmp/flux.tar.gz "https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" +tar -xzf /tmp/flux.tar.gz -C ~/.local/bin flux +rm /tmp/flux.tar.gz +flux --version +``` + +## 2. Bootstrap Flux against Forgejo + +Forgejo isn't a Flux-native provider the way GitHub/GitLab are, so this +uses the generic git bootstrap — over HTTPS with the `flux-write` token +from `docs/01-bootstrap.md` step 4, not SSH (Forgejo's git-SSH port turned +out not to be reliably reachable from either this desktop or the T630 — +see the earlier troubleshooting in this project's history): + +```sh +flux check --pre --kubeconfig ~/.kube/config-manual + +flux bootstrap git \ + --url=https://git.boglabob.com/codegit/cloud-demo \ + --branch=main \ + --path=clusters/homelab \ + --username=codegit \ + --password= \ + --token-auth \ + --kubeconfig ~/.kube/config-manual +``` + +`--password` here is the `flux-write` token, not an actual account +password. Flux stores it as a Kubernetes Secret in the `flux-system` +namespace once bootstrap completes — that Secret is what +`ImageUpdateAutomation` (step 7, below) reuses to push commits back, not +anything held by `k8s` itself. Clear this command from `k8s`'s shell +history afterwards (or prefix it with a space first, if +`HISTCONTROL=ignorespace` is set) since the token was passed as a plain +argument. + +This populates `clusters/homelab/flux-system/` and, because +`clusters/homelab/apps.yaml` already declares `Kustomization` objects for +`apps/podinfo`, `apps/hello-app`, and `apps/headlamp`, all three start +reconciling immediately. + +### What that one command actually did, mechanism by mechanism + +"Flux" isn't one program watching your repo by magic — it's a handful of +ordinary Kubernetes controllers (just Deployments, like anything else +you'd run on the cluster), each one understanding a couple of **Custom +Resource Definitions** — CRDs extend the Kubernetes API with new object +kinds, the same way `Deployment` or `Service` are built-in kinds. +`GitRepository` and `Kustomization` are two such kinds Flux adds. Nothing +about any of this is special-cased outside the normal Kubernetes API — +it's the exact same "define an object, a controller notices it and acts" +loop that runs the whole rest of Kubernetes. + +That one `flux bootstrap` command did five genuinely separate things. Go +look at each — these commands work against the cluster you already +bootstrapped: + +**1. It installed the controllers** — plain Kubernetes Deployments, no +different in kind from anything else running on the cluster: + +```sh +kubectl -n flux-system get deployments +``` + +`source-controller`, `kustomize-controller`, `helm-controller`, +`notification-controller`. Alongside them, it registered the CRDs those +controllers understand: + +```sh +kubectl get crds | grep fluxcd +``` + +**2. It created a `GitRepository` object** — this is the whole "what repo +am I watching" declaration, and nothing more. `source-controller`'s job is +entirely mechanical: every `interval` (default 1m), do a real `git fetch` +against `.spec.url`/`.spec.ref`; if the commit SHA changed, package that +tree into a `.tar.gz`, and record its location in `.status.artifact`. +That's the entire job — it doesn't know or care what's *in* the repo. + +```sh +kubectl -n flux-system get gitrepository flux-system -o yaml +``` + +Look at `.status.conditions` and `.status.artifact` — that's the result of +an actual git fetch that already happened, not a static config. + +**3. It created a `Kustomization` object** (confusingly, the same name as +the `kustomization.yaml` files already sitting in `apps/podinfo/` etc. — +related but not identical). `kustomize-controller`'s job, on its own +interval: fetch the artifact `source-controller` produced, run the real +`kustomize` tool against `.spec.path` inside it (the exact same tool a +plain `kustomize build apps/podinfo` would run locally against those +`kustomization.yaml` files), and apply the resulting objects via the +Kubernetes API — the automated equivalent of you running +`kubectl apply -f <(kustomize build apps/podinfo)` yourself, on a timer, +forever. `prune: true` (set on all the `Kustomization` objects in this +repo) means it also *deletes* anything it previously created that's no +longer present in the current git state — that's what makes it +self-healing rather than just "apply once." + +```sh +kubectl -n flux-system get kustomization +``` + +**This is also why `apps/podinfo`, `apps/hello-app`, and `apps/headlamp` +started deploying without you ever running `kubectl apply` on them.** +Bootstrap's own `Kustomization` watches `clusters/homelab` with +`prune: true`. `clusters/homelab/apps.yaml` (already sitting in the repo) +itself just *defines more `Kustomization` objects*, one per app — so the +first one picks it up as part of its own normal reconcile, creates those +three child `Kustomization`s, and each of *those* then does its own +fetch-and-apply against its own app directory. It's the same mechanism +recursing, not a special case. + +**4. It stored your credential as a Kubernetes Secret** — `--password` +here is the `flux-write` token, not an account password. Bootstrap +base64-wraps it into a Secret that matches what the `GitRepository`'s +`.spec.secretRef` points at: + +```sh +kubectl -n flux-system get secret flux-system -o yaml +``` + +The `password` field is base64 (`| base64 -d` to read it) — this is +exactly the same credential `git clone https://user:token@host/repo` would +use, just read by `source-controller` on every fetch instead of typed by +you once. It's never written to `k8s`'s own filesystem. + +**5. It committed that config back into the repo itself** — `git pull` in +your desktop clone and look at `clusters/homelab/flux-system/`. The +`GitRepository`/`Kustomization` objects you just inspected live in the +cluster *because* those exact YAML files are committed there — so +rebuilding this cluster from scratch would mean running `flux bootstrap` +again (or even just `kubectl apply -f clusters/homelab/flux-system/`) and +landing in the identical state. The fact that Flux watches this repo is +itself declared *in* this repo. (Stage 4 leans on exactly this property — +re-bootstrapping against a brand new cluster reproduces the same deployed +state with no manual replay.) + +If you want to watch a reconcile happen live rather than just inspect the +end state: + +```sh +kubectl -n flux-system logs deploy/source-controller -f +``` + +then in another terminal, make any commit and push it — you'll see the +next fetch pick it up within the interval. + +## 3. Verify the GitOps loop actually works + +```sh +flux get kustomizations --watch +``` + +Watch until `podinfo`, `hello-app`, and `headlamp` all show `Ready: True`, +then confirm pods actually landed: + +```sh +kubectl -n podinfo get pods +kubectl -n hello-app get pods +kubectl -n headlamp get pods +``` + +If that's all healthy, the entire chain — Forgejo repo → Flux → this +cluster — is working end to end with nothing manually `kubectl apply`'d. + +## 4. podinfo: Flux's first Helm-managed app + +podinfo is deployed first deliberately: it needs nothing this project +builds itself — no CI, no custom image, just a public chart — so a green +`flux get kustomization podinfo` proves the whole +source-controller → helm-controller → cluster loop works before hello-app +adds a CI dependency on top of it, and before headlamp adds a +cluster-admin RBAC concern on top of *that*. + +A **Helm chart** is a packaged bundle of Kubernetes YAML templates plus a +`values.yaml` of defaults — installing one produces the same kind of +`Deployment`/`Service`/etc. objects you'd otherwise hand-write, just +parameterized and versioned. Used directly (outside this project, with +nothing GitOps involved), that's a two-step, one-shot, imperative +process: + +```sh +helm repo add podinfo https://stefanprodan.github.io/podinfo # register the chart index +helm install podinfo podinfo/podinfo --set replicaCount=1 # render + apply, once +``` + +Nothing here uses the `helm` CLI at all — Flux replaces both of those +steps with two declarative objects that `helm-controller` reconciles +continuously instead of once: + +- **`apps/podinfo/helmrepository.yaml`** (a `HelmRepository`) is the + declarative form of `helm repo add` — it just tells `source-controller` + where the chart index lives and how often to refresh it (`interval`). + On its own it deploys nothing. +- **`apps/podinfo/helmrelease.yaml`** (a `HelmRelease`) is the declarative + form of `helm install`/`helm upgrade` — its `values:` block is exactly + what would otherwise be `-f values.yaml`/`--set` flags on the CLI, and + its `chart.spec.version` range (`>=6.0.0`) is what keeps it current: + `helm-controller` re-installs whenever a new matching chart version + shows up in the repository, on its own `interval`, with no one running + `helm upgrade` by hand. + +Both files already have comments walking through their specific fields — +worth reading now that you know what problem they're solving. + +## 5. headlamp: a cluster-admin UI, accessed on demand + +`apps/headlamp/` is deployed the same Helm-via-Flux way as podinfo (see +its `helmrepository.yaml`/`helmrelease.yaml`), plus one more piece: +`apps/headlamp/rbac.yaml` creates a `ServiceAccount` bound to the +`cluster-admin` `ClusterRole`, with a long-lived token Secret. Headlamp +grants whatever its logged-in identity can do — with that token, that's +full cluster-admin. (This project used the official Kubernetes Dashboard +originally; it's since been swapped for Headlamp, which the Dashboard +project itself now points people toward, but the security posture is +identical.) + +Publicly exposing a cluster-admin UI (even behind a login page) is the +exact pattern behind real breaches (Tesla, 2018: an internet-reachable, +unauthenticated Dashboard). So: no ingress, no standing hostname — only a +port-forward you open when you need it and close when you don't: + +```sh +kubectl -n headlamp port-forward svc/headlamp 8443:80 +``` + +Headlamp's chart doesn't terminate TLS on its own Service (that's left to +whatever fronts it in a real deployment — here, nothing does, since this +never leaves `localhost`), so open plain `http://localhost:8443`, not +`https://`. Log in by pasting the token: + +```sh +kubectl -n headlamp get secret admin-user-token -o jsonpath='{.data.token}' | base64 -d +``` + +## 6. Register the Forgejo Actions runner + +hello-app (next) is the only app in this repo whose image *this project's +own CI* builds — that needs a self-hosted Forgejo Actions runner; there's +no shared runner pool. The runner normally gets root-equivalent power over +its host by mounting `/var/run/docker.sock` (anyone who can push a +workflow file effectively gets root there). Instead: it runs as the same +unprivileged `k8s` user from `docs/01-bootstrap.md` step 2, using rootless +Podman's own socket instead of Docker's — no root anywhere in this +pipeline. `.forgejo/workflows/build-hello-app.yml` already builds images +with `kaniko`, which needs no daemon and no elevated privileges at all. + +The runner's job containers (kaniko, opentofu once stage 4 wires that +workflow in too) never get the libvirt socket or `k8s`'s home directory +mounted in — only the Podman socket, needed to launch those job containers +in the first place — so a compromised workflow can spawn containers as +`k8s`, but can't directly touch the VMs or Terraform state. Worth knowing +given the runner lives on the same box/user as the cluster's own +infrastructure; fine for a demo-sized project, but if this ever handles +anything sensitive, move the runner to its own user or VM so a breakout +doesn't share a blast radius with the cluster. + +1. As `k8s` (`ssh k8s@t630`), enable the rootless Podman API socket: + ```sh + systemctl --user enable --now podman.socket + echo $XDG_RUNTIME_DIR # note this path, e.g. /run/user/1001 + ``` +2. Instance admin: `Site Administration → Actions → Runners`, confirm + Actions is enabled. +3. Repo: `Settings → Actions → Runners → Create new runner`, copy the + registration token. +4. Register and run the runner as a rootless Podman container, pointed at + the Podman socket from step 1 instead of docker.sock: + ```sh + # still as k8s + podman volume create forgejo-runner-data + podman run -d --name forgejo-runner --restart unless-stopped \ + -e DOCKER_HOST="unix://$XDG_RUNTIME_DIR/podman/podman.sock" \ + -v "$XDG_RUNTIME_DIR/podman/podman.sock:$XDG_RUNTIME_DIR/podman/podman.sock" \ + -v forgejo-runner-data:/data \ + code.forgejo.org/forgejo/runner:6 \ + forgejo-runner register --no-interactive \ + --instance https://git.boglabob.com \ + --token --labels docker:docker://node:20-bookworm + ``` + The registered runner picks up both workflows in `.forgejo/workflows/` — + `terraform.yml`'s `container:` image (stage 4) and + `build-hello-app.yml`'s kaniko image (next) are both launched through + that same rootless Podman socket. +5. Repo `Settings → Secrets and Variables → Actions`, add: + - Secret `FORGEJO_TOKEN` — a personal access token (`Settings → Applications` + on your Forgejo user, scope `package:write`) used to push images. + - Variable `FORGEJO_USER`, `FORGEJO_ORG` — your Forgejo username/org. + +If the `hello-app` package ends up private (Forgejo package visibility +follows repo visibility by default), create the cluster-side pull secret +and uncomment the `imagePullSecrets` line in `apps/hello-app/deployment.yaml`: + +```sh +kubectl -n hello-app create secret docker-registry forgejo-registry \ + --docker-server=git.boglabob.com \ + --docker-username= \ + --docker-password= +``` + +## 7. hello-app: plain manifests, CI, and image automation + +Unlike podinfo/headlamp, `apps/hello-app/` isn't a chart — it's *this* +project's own code (source + Dockerfile under `apps/hello-app/src/`), so +there's no upstream chart to track and plain +`Deployment`/`Service`/`Ingress` manifests are simpler to read for +something this small (see `apps/hello-app/deployment.yaml`'s comments). +What it does need, that podinfo/headlamp don't, is a way to notice when CI +publishes a new image and roll it out — that's +`apps/hello-app/image-automation.yaml`'s job +(`ImageRepository`/`ImagePolicy`/`ImageUpdateAutomation`, all commented in +that file). + +Exercise the whole loop end to end: + +```sh +sed -i 's/This page is served from it\./This page is served from it — and this line proves it: edited via git push./' apps/hello-app/src/index.html +git add apps/hello-app/src/index.html +git commit -m "test the pipeline" +git push +``` + +Watch: `build-hello-app` runs in Forgejo Actions (which also stamps the +page with the current commit SHA and build time — see +`apps/hello-app/src/index.html`) → pushes a new tag to +`git.boglabob.com/codegit/hello-app` → Flux's `ImageRepository` picks it up +within a minute → `ImageUpdateAutomation` commits the new tag back to +`apps/hello-app/deployment.yaml` → the `hello-app` Kustomization +reconciles → `kubectl -n hello-app get pods` shows a new pod, and +`https://hello.boglabob.com` shows the new commit SHA/badge (once step 8 +below exposes it). + +## 8. Expose podinfo/hello-app through Caddy + +k3s's bundled ingress controller (Traefik) is already listening on this +node's own IP, port 80, routing by the `Host:` header from each app's +`Ingress` resource (already defined in `apps/podinfo` and +`apps/hello-app`). Caddy just needs to forward matching requests there — +headlamp and the k3s API server are deliberately **not** exposed this way; +step 5 above and `docs/04-tofu.md` step 5 cover why and how to reach them +instead. + +Two DNS records, as CNAMEs pointed at `git.boglabob.com` rather than +duplicating its IP directly — CNAME means "this name is an alias for that +one," so there's one place (that record) to update if the underlying IP +ever changes, instead of several: + +``` +podinfo.boglabob.com CNAME git.boglabob.com +hello.boglabob.com CNAME git.boglabob.com +``` + +Then in Caddy's own config, using the VM's pinned IP from stage 2: + +``` +podinfo.boglabob.com { + reverse_proxy http://:80 +} + +hello.boglabob.com { + reverse_proxy http://:80 +} +``` + +```sh +podman exec caddy reload --config /etc/caddy/Caddyfile +``` + +```sh +curl https://podinfo.boglabob.com/ +curl https://hello.boglabob.com/ +``` + +`docs/Caddyfile.example` has the multi-node version of these same blocks — +stage 4 points you back at it once there are three node IPs to +load-balance across instead of one. + +Next: `docs/04-tofu.md` — everything above still works exactly as-is; that +stage replaces the one hand-built VM under it with a Terraform-provisioned +3-node cluster and points this same Flux config at it. diff --git a/docs/04-tofu.md b/docs/04-tofu.md new file mode 100644 index 0000000..a530418 --- /dev/null +++ b/docs/04-tofu.md @@ -0,0 +1,274 @@ +# Stage 4: Graduating to Terraform/OpenTofu + +Stages 2-3 got you a real, working cluster with Flux managing every app in +this repo — on one hand-built VM. This stage replaces that VM with a +proper 3-node cluster provisioned by Terraform/OpenTofu via the +`dmacvicar/libvirt` provider, then points the exact same Flux config at +it. Nothing in `clusters/homelab/` or `apps/` changes — that's the point: +this repo's GitOps state was never tied to *how* the cluster under it got +built. + +You'll need `docs/01-bootstrap.md` steps 3-4: the `k3s_token` secret +(generated but unused until now) and the `k8s-readonly` token. + +## Where this track actually stands + +Earlier notes on this project flagged this whole track as "blocked" on a +provider schema mismatch between the 0.8.x and 0.9.x lines of +`dmacvicar/libvirt`, without pinning down exactly what broke. That's worth +re-examining rather than taking on faith, both because it matters for +whether you should trust `main.tf` and because the process of checking it +is itself a reasonable thing to learn from — so here's what's actually +been confirmed, and what hasn't, as of this pass: + +**Confirmed: this isn't really a "0.8.x vs 0.9.x" ambiguity at all.** +`terraform/versions.tf` pins `~> 0.8`. Checking the provider's own release +history: v0.9.0 (Nov 2025) was an intentional, permanent fork to a fully +regenerated schema that maps 1:1 onto libvirt's XML — the maintainer's own +release notes say so explicitly, and describe keeping 0.8.x alive in +parallel specifically for people who don't want that rewrite. So `~> 0.8` +doesn't risk drifting onto 0.9.x schema by accident; it's a deliberate, +stable choice, not an unresolved question. + +**Confirmed: every resource in `main.tf` matches the real 0.8.x schema.** +Checked directly against this provider's own docs at git tag `v0.8.3` +(`website/docs/r/{pool,volume,cloudinit,domain,network}.html.markdown` in +`dmacvicar/terraform-provider-libvirt`) — not from memory, and not from +whatever an LLM's training data assumes a "libvirt provider" looks like, +which is the trap the project's own earlier notes were rightly worried +about. Every attribute `main.tf` uses lines up: `libvirt_pool`'s +`type = "dir"`; `libvirt_volume`'s `base_volume_id`/`size`/`format`; +`libvirt_cloudinit_disk`'s `user_data`/`network_config`; +`libvirt_domain`'s `disk { volume_id }`, `network_interface { network_id, +wait_for_lease }`, and `console { type, target_type, target_port }`; +`libvirt_network`'s `mode`/`addresses`/`dhcp { enabled }`/`dns { enabled }`. +None of it uses 0.9.x-only shapes (nested `create.content.url`, +`backing_store`, `capacity` instead of `size`, etc.). + +**Not yet confirmed: whether it actually applies.** Static schema-matching +isn't the same as a real `tofu apply` succeeding — that needs the T630's +actual libvirt socket, which nothing has exercised end-to-end yet. If +you're picking this stage up, this is the real remaining unknown, and a +reasonable order to close it: + +1. **Free, no-VM checkpoint first**: `.forgejo/workflows/terraform.yml` + already runs `tofu init -backend=false` + `tofu validate` on every push + to `terraform/**` — that's a real, automated check of exactly the + schema question above, running today. Check its latest result in + Forgejo Actions before doing anything else; if it's failing, the error + message will point at a specific resource/attribute far faster than + re-deriving the whole schema by hand. +2. **`tofu init` on the T630** (step 2 below) and check + `terraform/.terraform.lock.hcl` afterwards — confirm it actually + resolved a `0.8.x` version, not something unexpected. +3. **`tofu plan`, then `tofu apply`**, and if any single resource fails, + treat that resource in isolation: re-check its specific arguments + against `website/docs/r/.html.markdown` at whatever version + `.terraform.lock.hcl` actually resolved (not `main` — the docs move + with the schema, and `main` may already reflect a newer 0.8.x patch or + even post-fork changes), rather than guessing at a fix. That's a more + targeted version of the same check already done above for the whole + file. +4. Once a full `tofu apply` succeeds once, the remaining steps below are + what actually plug the result into the rest of this project. + +Sizing is deliberately small (3 VMs, 2 vCPU/2GB RAM each = 6 vCPU/6GB +total) so this stays a demo rather than competing with whatever else is +already running on the T630. + +--- + +## 1. Install OpenTofu + +Installing the package needs sudo, so that part is you (`maq`), not `k8s`. +Installing system-wide (`/usr/local/bin`) means `k8s` can just use `tofu` +afterwards with no further root involvement: + +```sh +# as maq (has sudo) +sudo apt install -y unzip +curl -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh +sudo sh install-opentofu.sh --install-method standalone && rm install-opentofu.sh +``` + +## 2. Provision the VMs with OpenTofu + +Everything from here on is `k8s` again (`sudo -iu k8s`), no sudo involved — +clone using the `k8s-readonly` token from `docs/01-bootstrap.md` step 4. +`k8s` has no keyring (it's headless, no desktop session), so this uses +`git credential-store` — a plaintext file, `chmod 600`'d, holding only the +read-only token: + +```sh +# as k8s +git config --global credential.helper store +git clone https://git.boglabob.com/codegit/cloud-demo.git ~/k3s +# prompts for username (anything) and password (paste the k8s-readonly +# token) once; stores it in ~/.git-credentials for next time +chmod 600 ~/.git-credentials + +cd ~/k3s/terraform +cp terraform.tfvars.example terraform.tfvars +# edit terraform.tfvars: ssh_public_key (contents of ~/.ssh/id_ed25519.pub +# from docs/01-bootstrap.md step 2), k3s_token (from step 3). Defaults for +# network/sizing are fine to start. + +tofu init +tofu plan +tofu apply +``` + +For any later change to `terraform/`: edit and push as `maq` as usual, then +`cd ~/k3s && git pull` as `k8s` before re-running `tofu plan`/`apply`. + +This brings up `k3s-server-1`, `k3s-agent-1`, `k3s-agent-2` on the +`k3s-homelab` libvirt network (`10.20.30.0/24` by default — isolated from +anything else already using libvirt on this box, including the `default` +network stage 2's manual VM used). Cloud-init installs k3s on each on +first boot — give it ~2 minutes after `apply` finishes. + +## 3. Get kubectl talking to the new cluster + +```sh +mkdir -p ~/.kube +ssh k3s@$(tofu output -raw server_ip) sudo cat /etc/rancher/k3s/k3s.yaml \ + | sed "s/127.0.0.1/$(tofu output -raw server_ip)/" > ~/.kube/config-homelab + +export KUBECONFIG=~/.kube/config-homelab +kubectl get nodes # expect 3 Ready nodes +``` + +This works as-is because you're running it on the T630, which can reach the +`10.20.30.0/24` network directly. To also use kubectl from your own laptop, +either `scp` this kubeconfig over and open an SSH tunnel first +(`ssh -L 6443:10.20.30.11:6443 k8s@t630`, then point the kubeconfig's +`server:` at `https://127.0.0.1:6443`), or just SSH into the T630 as `k8s` +whenever you need kubectl — simplest by far for a project this size. + +## 4. Re-bootstrap Flux against the new cluster + +This is the exact same `flux bootstrap git` command from `docs/03-flux.md` +step 2, just pointed at `~/.kube/config-homelab` instead of +`~/.kube/config-manual` — and it needs to actually run again, not be +skipped. The new 3-node cluster has its own fresh etcd/SQLite; nothing +about the manual VM's cluster state carries over to it, `flux-system` +namespace included. What *does* carry over is this repo: + +```sh +flux check --pre --kubeconfig ~/.kube/config-homelab + +flux bootstrap git \ + --url=https://git.boglabob.com/codegit/cloud-demo \ + --branch=main \ + --path=clusters/homelab \ + --username=codegit \ + --password= \ + --token-auth \ + --kubeconfig ~/.kube/config-homelab +``` + +Because `clusters/homelab/flux-system/` already holds the exact manifests +stage 3's bootstrap generated, this run doesn't need to commit anything +new back to the repo — it just applies that already-correct config to a +cluster that doesn't have it yet. That's the whole point of GitOps having +been the deploy mechanism all along: the desired state was never tied to +the specific cluster instance, so pointing the same bootstrap command at a +new kubeconfig reproduces it exactly. Confirm: + +```sh +flux get kustomizations --watch +kubectl -n podinfo get pods +kubectl -n hello-app get pods +kubectl -n headlamp get pods +``` + +## 5. kubectl access from elsewhere on the LAN, or remotely + +Best practice for the Kubernetes API server is the same principle as +Headlamp (`docs/03-flux.md` step 5): never put 6443 on the public +internet if you can avoid it, because a leaked credential there is a full +cluster compromise. + +Node IPs (`10.20.30.0/24` by default) live on the private libvirt network +from step 2 — only the T630 itself can reach them directly, which is +actually a nice side effect: even the rest of your LAN can't touch the API +server without going through the T630 first. Two ways to do that: + +- **SSH into the T630 as `k8s`** and run kubectl there directly (same as + step 3) — simplest, and what this whole project assumes by default. +- **Tunnel from another machine** (your laptop, or a phone via Termux, etc.): + ```sh + ssh -L 6443:$(tofu output -raw server_ip):6443 k8s@t630 + ``` + then point a local kubeconfig's `server:` at `https://127.0.0.1:6443` + (copy the kubeconfig from step 3 and edit that one field). The cert + validates because `k8s-api.boglabob.com` is in the server's TLS SAN list + (`terraform/variables.tf`'s `k8s_api_hostname`) — add it to + `/etc/hosts` as `127.0.0.1 k8s-api.boglabob.com` on whatever machine + you're tunneling from and use that as the `server:` host instead of the + raw IP, so the hostname in the URL matches a name the cert actually + covers. (This SAN entry is new in stage 4 — the manual VM's cloud-init + in `docs/02-k3s.md` never set one, since nothing needed LAN-wide access + to it.) +- **From outside your home network entirely**: Tailscale or WireGuard on + the T630, then the SSH tunnel above over the Tailscale/WireGuard link + instead of the open internet. Reasonable next stretch goal once the core + loop is working — don't port-forward 22 or 6443 on your router for this. + +The `admin-user` bearer token (`docs/03-flux.md` step 5) also works for +kubectl over the same tunnel, if you'd rather not manage the client-cert +kubeconfig. + +## 6. Point Caddy and Headlamp at the new cluster + +Same steps as `docs/03-flux.md` step 8 (Caddy) and step 5 (Headlamp) — the +mechanism is identical, only the node IP(s) changed. Caddy now has three +node IPs to pick from instead of one, so use `docs/Caddyfile.example`'s +`reverse_proxy` blocks (which list all three) rather than the single-IP +version from stage 3: + +```sh +tofu output node_ips +``` + +```sh +podman exec caddy reload --config /etc/caddy/Caddyfile +``` + +Headlamp's port-forward command is unchanged (`kubectl` just needs +`KUBECONFIG` pointed at `~/.kube/config-homelab` now). + +## 7. Tearing down the manual VM + +Now that the real 3-node cluster is up, remove stage 2's throwaway one: + +```sh +virsh -c qemu:///system destroy k3s-manual # stop it +virsh -c qemu:///system undefine k3s-manual --remove-all-storage # VM + overlay disk +virsh -c qemu:///system vol-delete --pool default k3s-manual-base.qcow2 # base image isn't + # attached to the VM + # directly, needs its + # own delete +rm ~/.kube/config-manual +rm -rf ~/vms +``` + +Nothing else needs cleaning up — Flux's own state lived entirely inside +that VM's cluster and goes away with it. The Forgejo repo, both tokens, +and the Forgejo Actions runner registration are all cluster-independent +and already carried over unchanged in step 4 above. + +## Stretch goals, roughly in order + +- **Remote access**: Tailscale or WireGuard on the k3s server node, for + kubectl/Headlamp access from outside the LAN without opening anything + publicly (step 5). +- **TLS**: `cert-manager` + a `ClusterIssuer` for Let's Encrypt (DNS-01 if + `boglabob.com` isn't publicly reachable on 80/443). +- **Secrets in Git**: `sops` + `age`, or `sealed-secrets`, so the + `K3S_TOKEN`/API tokens above don't need to live only in Forgejo's secret + store. +- **Monitoring**: `kube-prometheus-stack` via Helm, deployed the same way as + podinfo (HelmRepository + HelmRelease under `apps/`). +- **HA**: add a second k3s server node and switch from SQLite to embedded + etcd (`--cluster-init` on the first server, `--server` join on the second). diff --git a/docs/Caddyfile.example b/docs/Caddyfile.example index da6abc7..8d1f9f0 100644 --- a/docs/Caddyfile.example +++ b/docs/Caddyfile.example @@ -14,12 +14,12 @@ # forward the request (Host header included) to any node. Listing all three # gives you free load-balancing/failover across nodes. # -# Deliberately NOT here: the Kubernetes Dashboard and the k3s API server. -# Both grant cluster-admin-level control, and routing either through a -# public-facing reverse proxy is the exact pattern behind real-world -# cluster breaches (e.g. Tesla, 2018 — an exposed, unauthenticated -# Dashboard). Both stay LAN-only / on-demand instead — see docs/SETUP.md -# steps 12-13. +# Deliberately NOT here: Headlamp and the k3s API server. Both grant +# cluster-admin-level control, and routing either through a public-facing +# reverse proxy is the exact pattern behind real-world cluster breaches +# (e.g. Tesla, 2018 — an exposed, unauthenticated Dashboard). Both stay +# LAN-only / on-demand instead — see docs/03-flux.md step 5 (Headlamp) and +# docs/04-tofu.md step 5 (kubectl/API access). podinfo.boglabob.com { reverse_proxy http://10.20.30.11:80 http://10.20.30.12:80 http://10.20.30.13:80 diff --git a/docs/SETUP.md b/docs/SETUP.md deleted file mode 100644 index 40fca34..0000000 --- a/docs/SETUP.md +++ /dev/null @@ -1,401 +0,0 @@ -# Setup walkthrough (full, Terraform-driven) - -This is the "do it properly" path — Terraform/OpenTofu provisioning all 3 -VMs via the `dmacvicar/libvirt` provider. It's also the one currently -blocked on getting that provider's HCL right (0.8.x vs 0.9.x schema — see -conversation history). If you want a real cluster to learn Flux/GitOps on -*right now* without waiting on that, see `docs/QUICKSTART.md` instead — a -single manually-created VM, no Terraform, with teardown instructions for -switching over once this track is sorted. Steps 1–4 below are shared -between both guides. - -Assumes: the T630 is an existing Debian box already running other -self-hosted services — this project installs alongside those as ordinary -packages (`qemu-kvm`/`libvirt`), not a hypervisor OS replacing Debian, and -is sized deliberately small (3 VMs, 2 vCPU/2GB RAM each = 6 vCPU/6GB total) -so it stays a demo rather than competing with what's already running. Forgejo -is already running and reachable at `https://git.boglabob.com`, and you can -point DNS records under `boglabob.com` at hosts on your network (directly, -or via whatever reverse proxy/tunnel already gets `git.boglabob.com` there). - ---- - -## 1. Install KVM/libvirt on the T630 - -Ordinary packages, no reboot into an installer, nothing else on the box is -touched: - -```sh -# on the T630 -sudo apt update -sudo apt install -y qemu-kvm libvirt-daemon-system libvirt-clients virtinst - -# confirm hardware virtualization is available (T630's Xeons support it) -sudo kvm-ok -``` - -## 2. Create the unprivileged 'k8s' user - -One dedicated, no-sudo user for everything this project touches: driving -Terraform/kubectl/flux against libvirt here, and running the Forgejo Actions -runner later (step 9). It needs group membership to talk to libvirt — that's -a one-time root action; nothing it does afterwards needs `sudo`. - -```sh -sudo useradd -m -s /bin/bash k8s # one-time, needs root to create the user itself -sudo usermod -aG libvirt,kvm k8s -sudo loginctl enable-linger k8s # lets its services keep running after logout - -# as k8s, from here on (sudo -iu, not su -, since k8s has no password set): -sudo -iu k8s -ssh-keygen -t ed25519 -C "k3s-homelab" -f ~/.ssh/id_ed25519 # only needed if you'll SSH in as k8s day-to-day -virsh -c qemu:///system list --all # sanity check: should run with no permission error, no sudo -``` - -Do the rest of this guide logged in as `k8s` on the T630 itself (`ssh -k8s@t630`) — node IPs (step 5) live on a private libvirt network that's only -directly reachable from the T630, so this is the simplest place to run -`tofu`/`kubectl`/`flux` from. (If you'd rather drive Terraform from your own -workstation instead, see the `libvirt_uri` comment in -`terraform/terraform.tfvars.example` — you'll then need an SSH tunnel for -kubectl/flux to reach node IPs.) - -## 3. Generate the secrets Terraform needs - -```sh -openssl rand -hex 32 # -> k3s_token -``` - -## 4. Push this repo to Forgejo - -Doing this before provisioning (rather than after) means `k8s` can get the -repo with a plain `git clone` in step 5, instead of needing a one-off copy -handed to it — and any future Terraform change just needs a `git pull`. -Repo/owner used throughout this guide: `codegit/cloud-demo` (already baked -into `apps/hello-app/deployment.yaml` and `image-automation.yaml`'s image -references — no placeholder-swapping needed). - -1. On Forgejo (`https://git.boglabob.com`), as `codegit`: **+ → New - Repository** → name `cloud-demo`. Leave it empty — don't initialize with - a README/`.gitignore`/license, since this repo already has its own. - Visibility (public/private) is your call; either works, since access for - `k8s`/Flux/CI goes through the tokens below regardless. -2. Locally, wherever you're editing this repo (`maq`): - ```sh - git init # if not already - git add . - git commit -m "initial scaffold" - git remote add origin https://git.boglabob.com/codegit/cloud-demo.git - git push -u origin main - ``` -3. Generate two access tokens (`Settings → Applications → Generate New - Token`), scoped as narrowly as Forgejo's token UI allows to repository - read/write: - - **`k8s-readonly`** — read-only. Used only for `k8s`'s own manual - `git clone`/`pull` on the T630 (step 5) — never leaves that box, isn't - used by anything automated. - - **`flux-write`** — read/write. Used once, as a `flux bootstrap` - argument (step 7); Flux stores it as a Kubernetes Secret inside the - cluster from then on (`ImageUpdateAutomation`'s commits back in step 10 - reuse that same in-cluster Secret) — it's never written to `k8s`'s - filesystem at all. - - Using HTTPS tokens instead of `k8s`'s SSH key (`~/.ssh/id_ed25519`, from - step 2) sidesteps an open question: Forgejo's git-SSH port isn't - reachable from this desktop through your router (see the SSH - troubleshooting earlier in this conversation), and whether it's reachable - from `k8s` on the T630 itself was never actually confirmed either. HTTPS - (443, via Caddy) is already proven to work, so both tokens use that - instead. Copy both token values now — Forgejo only shows them once. - -## 5. Provision the VMs with OpenTofu - -Installing the package needs sudo, so that part is you (`maq`), not `k8s`. -Installing system-wide (`/usr/local/bin`) means `k8s` can just use `tofu` -afterwards with no further root involvement: - -```sh -# as maq (has sudo) -sudo apt install -y unzip -curl -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh -sudo sh install-opentofu.sh --install-method standalone && rm install-opentofu.sh -``` - -Everything from here on is `k8s` again (`sudo -iu k8s`), no sudo involved — -clone using the `k8s-readonly` token from step 4. `k8s` has no keyring (it's -headless, no desktop session), so this uses `git credential-store` — a -plaintext file, `chmod 600`'d, holding only the read-only token: - -```sh -# as k8s -git config --global credential.helper store -git clone https://git.boglabob.com/codegit/cloud-demo.git ~/k3s -# prompts for username (anything) and password (paste the k8s-readonly -# token) once; stores it in ~/.git-credentials for next time -chmod 600 ~/.git-credentials - -cd ~/k3s/terraform -cp terraform.tfvars.example terraform.tfvars -# edit terraform.tfvars: ssh_public_key (contents of ~/.ssh/id_ed25519.pub -# from step 2), k3s_token. Defaults for network/sizing are fine to start. - -tofu init -tofu plan -tofu apply -``` - -For any later change to `terraform/`: edit and push as `maq` as usual, then -`cd ~/k3s && git pull` as `k8s` before re-running `tofu plan`/`apply`. - -This brings up `k3s-server-1`, `k3s-agent-1`, `k3s-agent-2` on the -`k3s-homelab` libvirt network (`10.20.30.0/24` by default — isolated from -anything else already using libvirt on this box). Cloud-init installs k3s on -each on first boot — give it ~2 minutes after `apply` finishes. - -## 6. Get kubectl talking to the cluster - -`kubectl` itself was never actually installed anywhere earlier in this -guide despite being listed as a prerequisite — install it now (as `k8s`, -no sudo needed, same pattern as the OpenTofu install): - -```sh -curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -chmod +x kubectl -mkdir -p ~/.local/bin -mv kubectl ~/.local/bin/ -echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc -export PATH="$HOME/.local/bin:$PATH" -``` - -```sh -mkdir -p ~/.kube -ssh k3s@$(tofu output -raw server_ip) sudo cat /etc/rancher/k3s/k3s.yaml \ - | sed "s/127.0.0.1/$(tofu output -raw server_ip)/" > ~/.kube/config-homelab - -export KUBECONFIG=~/.kube/config-homelab -kubectl get nodes # expect 3 Ready nodes -``` - -This works as-is because you're running it on the T630, which can reach the -`10.20.30.0/24` network directly. To also use kubectl from your own laptop, -either `scp` this kubeconfig over and open an SSH tunnel first -(`ssh -L 6443:10.20.30.11:6443 k8s@t630`, then point the kubeconfig's -`server:` at `https://127.0.0.1:6443`), or just SSH into the T630 as `k8s` -whenever you need kubectl — simplest by far for a project this size. - -## 7. Bootstrap Flux against Forgejo - -Forgejo isn't a Flux-native provider (unlike GitHub/GitLab), so use the -generic git bootstrap — over HTTPS with the `flux-write` token from step 4, -not SSH (same reachability reasoning as step 5): - -Install the Flux CLI the same no-sudo, no-package-manager way as `kubectl` -(`brew` assumes Homebrew, which isn't a given on a bare Debian box): - -```sh -FLUX_VERSION=$(curl -s https://api.github.com/repos/fluxcd/flux2/releases/latest | grep tag_name | cut -d '"' -f4 | sed 's/^v//') -curl -L -o /tmp/flux.tar.gz "https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" -tar -xzf /tmp/flux.tar.gz -C ~/.local/bin flux -rm /tmp/flux.tar.gz -flux --version -``` - -```sh -flux check --pre --kubeconfig ~/.kube/config-homelab - -flux bootstrap git \ - --url=https://git.boglabob.com/codegit/cloud-demo \ - --branch=main \ - --path=clusters/homelab \ - --username=codegit \ - --password= \ - --token-auth \ - --kubeconfig ~/.kube/config-homelab -``` - -`--password` here is the `flux-write` token, not an actual account -password. Flux stores it as a Kubernetes Secret in the `flux-system` -namespace once bootstrap completes — that Secret is what -`ImageUpdateAutomation` (step 10) reuses to push commits back, not anything -held by `k8s` itself. Clear this command from `k8s`'s shell history -afterwards (or prefix it with a space first, if `HISTCONTROL=ignorespace` -is set) since the token was passed as a plain argument. - -This populates `clusters/homelab/flux-system/` and, because -`clusters/homelab/apps.yaml` already declares `Kustomization` objects for -`apps/podinfo` and `apps/hello-app`, both start reconciling immediately. - -## 8. Verify the podinfo GitOps loop - -```sh -flux get kustomizations --watch -kubectl -n podinfo get pods -``` - -Once it's `Ready`, point DNS at it and check in a browser (see step 11). - -## 9. Enable Forgejo Actions and register a runner (rootless, no sudo) - -Forgejo Actions needs a self-hosted runner — there's no shared runner pool. -The runner normally gets root-equivalent power over its host by mounting -`/var/run/docker.sock` (anyone who can push a workflow file effectively gets -root there). Instead: it runs as the same unprivileged `k8s` user from -step 2, using rootless Podman's own socket instead of Docker's — no root -anywhere in this pipeline. `build-hello-app.yml` already builds images with -kaniko, which needs no daemon and no elevated privileges at all. - -The runner's job containers (kaniko, opentofu) never get the libvirt socket -or `k8s`'s home directory mounted in — only the Podman socket, needed to -launch those job containers in the first place — so a compromised workflow -can spawn containers as `k8s`, but can't directly touch the VMs or -Terraform state. Worth knowing given the runner lives on the same box/user -as the cluster's own infrastructure; fine for a demo-sized project, but if -this ever handles anything sensitive, move the runner to its own user or -VM so a breakout doesn't share a blast radius with the cluster. - -1. As `k8s` (`ssh k8s@t630`), enable the rootless Podman API socket: - ```sh - systemctl --user enable --now podman.socket - echo $XDG_RUNTIME_DIR # note this path, e.g. /run/user/1001 - ``` -2. Instance admin: `Site Administration → Actions → Runners`, confirm - Actions is enabled. -3. Repo: `Settings → Actions → Runners → Create new runner`, copy the - registration token. -4. Register and run the runner as a rootless Podman container, pointed at - the Podman socket from step 1 instead of docker.sock: - ```sh - # still as k8s - podman volume create forgejo-runner-data - podman run -d --name forgejo-runner --restart unless-stopped \ - -e DOCKER_HOST="unix://$XDG_RUNTIME_DIR/podman/podman.sock" \ - -v "$XDG_RUNTIME_DIR/podman/podman.sock:$XDG_RUNTIME_DIR/podman/podman.sock" \ - -v forgejo-runner-data:/data \ - code.forgejo.org/forgejo/runner:6 \ - forgejo-runner register --no-interactive \ - --instance https://git.boglabob.com \ - --token --labels docker:docker://node:20-bookworm - ``` - The registered runner picks up both workflows in `.forgejo/workflows/` — - `terraform.yml`'s `container:` image and `build-hello-app.yml`'s kaniko - image are both launched through that same rootless Podman socket. -5. Repo `Settings → Secrets and Variables → Actions`, add: - - Secret `FORGEJO_TOKEN` — a personal access token (`Settings → Applications` - on your Forgejo user, scope `package:write`) used to push images. - - Variable `FORGEJO_USER`, `FORGEJO_ORG` — your Forgejo username/org. - -If the `hello-app` package ends up private (Forgejo package visibility -follows repo visibility by default), create the cluster-side pull secret and -uncomment the `imagePullSecrets` line in `apps/hello-app/deployment.yaml`: - -```sh -kubectl -n hello-app create secret docker-registry forgejo-registry \ - --docker-server=git.boglabob.com \ - --docker-username= \ - --docker-password= -``` - -## 10. Exercise the full loop - -```sh -sed -i 's/This page is served from it\./This page is served from it — and this line proves it: edited via git push./' apps/hello-app/src/index.html -git add apps/hello-app/src/index.html -git commit -m "test the pipeline" -git push -``` - -Watch: `build-hello-app` runs in Forgejo Actions (which also stamps the page -with the current commit SHA and build time — see `apps/hello-app/src/index.html`) -→ pushes a new tag to `git.boglabob.com/codegit/hello-app` → Flux's -`ImageRepository` picks it up within a minute → `ImageUpdateAutomation` -commits the new tag back to `apps/hello-app/deployment.yaml` → the -`hello-app` Kustomization reconciles → `kubectl -n hello-app get pods` shows -a new pod, and `https://hello.boglabob.com` shows the new commit SHA/badge. - -## 11. Expose the apps through Caddy - -Since Caddy (Podman) is already the front door for `git.boglabob.com`, route -`podinfo` and `hello-app` through it too — but not the Dashboard or the API -server; see steps 12-13 for why. - -1. Add CNAME records for `podinfo.boglabob.com` and `hello.boglabob.com` - pointing at `git.boglabob.com` (matching how every other record for this - server is set up) — one source of truth for the Caddy host's IP, rather - than duplicating it across records. -2. Add the blocks from `docs/Caddyfile.example` to Caddy's config, filling - in your real node IPs (`tofu output node_ips`), and reload: - ```sh - podman exec caddy reload --config /etc/caddy/Caddyfile - ``` -3. Check: - ```sh - curl https://podinfo.boglabob.com/ - curl https://hello.boglabob.com/ - ``` - -## 12. Access the Dashboard (LAN-only, on demand) - -The Dashboard grants whatever its logged-in identity can do — with the -`admin-user` token from `apps/kubernetes-dashboard/rbac.yaml`, that's -cluster-admin. Publicly exposing that (even behind a login page) is the -exact pattern behind real breaches (Tesla, 2018: an internet-reachable, -unauthenticated Dashboard). So: no ingress, no standing hostname — only a -port-forward you open when you need it and close when you don't: - -```sh -kubectl -n kubernetes-dashboard port-forward svc/kubernetes-dashboard-kong-proxy 8443:443 -``` - -Then open `https://localhost:8443` and log in with the token: - -```sh -kubectl -n kubernetes-dashboard get secret admin-user-token -o jsonpath='{.data.token}' | base64 -d -``` - -## 13. kubectl access from elsewhere on the LAN, or remotely - -Best practice for the Kubernetes API server is the same principle as the -Dashboard: never put 6443 on the public internet if you can avoid it, -because a leaked credential there is a full cluster compromise. - -Node IPs (`10.20.30.0/24` by default) live on the private libvirt network -from step 5 — only the T630 itself can reach them directly, which is -actually a nice side effect: even the rest of your LAN can't touch the API -server without going through the T630 first. Two ways to do that: - -- **SSH into the T630 as `k8s`** and run kubectl there directly (same as - step 6) — simplest, and what this whole guide assumes by default. -- **Tunnel from another machine** (your laptop, or a phone via Termux, etc.): - ```sh - ssh -L 6443:$(tofu output -raw server_ip):6443 k8s@t630 - ``` - then point a local kubeconfig's `server:` at `https://127.0.0.1:6443` - (copy the kubeconfig from step 6 and edit that one field). The cert - validates because `k8s-api.boglabob.com` is in the server's TLS SAN list - (`terraform/variables.tf`'s `k8s_api_hostname`) — add it to - `/etc/hosts` as `127.0.0.1 k8s-api.boglabob.com` on whatever machine - you're tunneling from and use that as the `server:` host instead of the - raw IP, so the hostname in the URL matches a name the cert actually - covers. -- **From outside your home network entirely**: Tailscale or WireGuard on - the T630, then the SSH tunnel above over the Tailscale/WireGuard link - instead of the open internet. Reasonable next stretch goal once the core - loop is working — don't port-forward 22 or 6443 on your router for this. - -The `admin-user` bearer token (step 12) also works for kubectl over the same -tunnel, if you'd rather not manage the client-cert kubeconfig. - -## Stretch goals, roughly in order - -- **Remote access**: Tailscale or WireGuard on the k3s server node, for - kubectl/Dashboard access from outside the LAN without opening anything - publicly (step 13). -- **TLS**: `cert-manager` + a `ClusterIssuer` for Let's Encrypt (DNS-01 if - `boglabob.com` isn't publicly reachable on 80/443). -- **Secrets in Git**: `sops` + `age`, or `sealed-secrets`, so the - `K3S_TOKEN`/API tokens above don't need to live only in Forgejo's secret - store. -- **Monitoring**: `kube-prometheus-stack` via Helm, deployed the same way as - podinfo (HelmRepository + HelmRelease under `apps/`). -- **HA**: add a second k3s server node and switch from SQLite to embedded - etcd (`--cluster-init` on the first server, `--server` join on the second). diff --git a/terraform/cloud-init/agent.yaml.tpl b/terraform/cloud-init/agent.yaml.tpl index 81bc5af..8079865 100644 --- a/terraform/cloud-init/agent.yaml.tpl +++ b/terraform/cloud-init/agent.yaml.tpl @@ -14,16 +14,30 @@ package_update: true packages: - curl +# A script instead of a plain runcmd line for one reason: Terraform (see +# main.tf's closing comment) brings all VMs up in parallel, with no +# ordering guarantee that the server finishes installing k3s before an +# agent tries to join it. Baking the retry loop into a script keeps that +# concern out of runcmd, which just calls it once. write_files: - path: /usr/local/bin/join-k3s.sh permissions: '0755' content: | #!/bin/sh # Server may still be booting; retry the join until it answers. + # /ping is k3s's own unauthenticated liveness endpoint — this only + # confirms the API server is accepting connections yet, it's not a + # credential check (K3S_TOKEN below is what actually authorizes the + # join once it happens). until curl -sk https://${server_ip}:6443/ping >/dev/null 2>&1; do echo "waiting for k3s server at ${server_ip}..." sleep 5 done + # Agents pass the token via environment variables to the installer + # rather than a config.yaml file (contrast server.yaml.tpl) because + # get.k3s.io's own install script reads K3S_URL/K3S_TOKEN directly + # for the "join an existing cluster" path — there's no separate + # agent config file it looks for the way the server has one. curl -sfL https://get.k3s.io | \ K3S_URL="https://${server_ip}:6443" \ K3S_TOKEN="${k3s_token}" \ diff --git a/terraform/cloud-init/network-config.yaml.tpl b/terraform/cloud-init/network-config.yaml.tpl index 72a62d4..52b56e5 100644 --- a/terraform/cloud-init/network-config.yaml.tpl +++ b/terraform/cloud-init/network-config.yaml.tpl @@ -1,3 +1,13 @@ +# cloud-init's network-config schema (distinct from the #cloud-config +# user-data schema in server.yaml.tpl/agent.yaml.tpl — no "#cloud-config" +# header here, and `version: 2` picks the netplan-style dialect). eth0 is +# the interface name libvirt's virtio NIC presents as inside a fresh +# Ubuntu cloud image — the only interface that exists at boot, since each +# node has exactly one network_interface in main.tf's libvirt_domain. +# `nameservers` points at the gateway rather than a public resolver +# because that's also where libvirt's own DNS forwarder listens (the `dns +# { enabled = true }` block on libvirt_network.k3s in main.tf) — it +# resolves both k3s.local addresses and forwards everything else out. version: 2 ethernets: eth0: diff --git a/terraform/cloud-init/server.yaml.tpl b/terraform/cloud-init/server.yaml.tpl index 53d9066..2829b5e 100644 --- a/terraform/cloud-init/server.yaml.tpl +++ b/terraform/cloud-init/server.yaml.tpl @@ -14,17 +14,34 @@ package_update: true packages: - curl +# k3s reads /etc/rancher/k3s/config.yaml automatically on install — this +# is the file, not command-line flags, specifically so the token never +# shows up in `ps`/process listings or shell history on the node, and +# `runcmd` below can stay a one-liner with no secrets in it. 0600 so only +# root can read it. write_files: - path: /etc/rancher/k3s/config.yaml permissions: '0600' content: | token: "${k3s_token}" + # Every hostname a client might use to reach this server needs to be + # in the API server's TLS certificate up front, or that client's TLS + # handshake fails outright — it can't be added after the fact + # without regenerating the cert. `${hostname}` covers this node's + # own name; `${k8s_api_hostname}` is the LAN-only name + # docs/04-tofu.md step 5 sets up for kubectl access from elsewhere on + # the network — both need to be here even though nothing uses the + # second one yet at boot time. tls-san: - "${hostname}" - "${k8s_api_hostname}" runcmd: - curl -sfL https://get.k3s.io | sh -s - server + # k3s's own kubeconfig is generated readable only by root (it grants + # full cluster-admin access) — this copies it somewhere the unprivileged + # `k3s` user can read, so docs/04-tofu.md step 3 can fetch it over SSH + # without needing root on the node. - mkdir -p /home/k3s/.kube - k3s kubectl config view --raw > /home/k3s/.kube/config - chown -R k3s:k3s /home/k3s/.kube diff --git a/terraform/main.tf b/terraform/main.tf index 02241f2..f1fb577 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -4,12 +4,24 @@ locals { prefix_length = split("/", var.network_cidr)[1] } +# A dedicated pool, separate from libvirt's own `default` — same +# "isolated from anything else on the host" motivation as the network +# below, and it's the fix for the storage-permissions problem +# docs/02-k3s.md walks through by hand ("Storage: why a raw path in ~ +# doesn't work"): a `dir` pool is a libvirtd-managed location, so volumes +# created in it come out owned correctly for the `libvirt-qemu` process +# that actually runs each VM, regardless of what user (`k8s`) asked for it. resource "libvirt_pool" "k3s" { name = var.storage_pool type = "dir" path = var.storage_pool_path } +# Downloaded once and shared as a read-only backing image for every node's +# own disk below (libvirt_volume.node's base_volume_id) — the same +# copy-on-write relationship docs/02-k3s.md explains in detail +# ("Building the base image..."). Terraform only re-downloads this if +# base_image_url changes; it doesn't redo it on every apply. resource "libvirt_volume" "base" { name = "k3s-base.qcow2" pool = libvirt_pool.k3s.name @@ -35,6 +47,10 @@ resource "libvirt_network" "k3s" { } } +# One overlay disk per node, all backed by the single shared base image +# above — `size` here is the overlay's logical capacity (what the guest +# OS sees), not how much host disk it actually consumes; qcow2 overlays +# start small and grow only as the node writes data. resource "libvirt_volume" "node" { for_each = var.nodes name = "${each.key}.qcow2" @@ -44,11 +60,18 @@ resource "libvirt_volume" "node" { format = "qcow2" } +# The NoCloud seed ISO for each node — this resource is the Terraform +# equivalent of `virt-install --cloud-init` in docs/02-k3s.md, just +# built from two separate templates instead of one hand-assembled file: resource "libvirt_cloudinit_disk" "node" { for_each = var.nodes name = "${each.key}-cloudinit.iso" pool = libvirt_pool.k3s.name + # "what should exist on this machine" — which cloud-init template + # (server vs agent) depends on each.value.role, so a server node and an + # agent node run entirely different runcmd payloads despite sharing this + # one resource block. See cloud-init/server.yaml.tpl and agent.yaml.tpl. user_data = templatefile("${path.module}/cloud-init/${each.value.role}.yaml.tpl", { hostname = each.key ssh_public_key = var.ssh_public_key @@ -57,6 +80,11 @@ resource "libvirt_cloudinit_disk" "node" { k8s_api_hostname = var.k8s_api_hostname }) + # "how should this machine's network be set up" — static, unlike stage + # 2's DHCP-based VM (docs/02-k3s.md), because this network + # (libvirt_network.k3s below) deliberately runs no DHCP server at all; + # every node needs to know its own address up front instead of waiting + # on a lease. network_config = templatefile("${path.module}/cloud-init/network-config.yaml.tpl", { ip = each.value.ip prefix_length = local.prefix_length @@ -64,6 +92,8 @@ resource "libvirt_cloudinit_disk" "node" { }) } +# The VM itself — everything above this point (pool, base image, per-node +# overlay, cloudinit ISO, network) exists only to be wired together here. resource "libvirt_domain" "node" { for_each = var.nodes name = each.key @@ -73,7 +103,10 @@ resource "libvirt_domain" "node" { cloudinit = libvirt_cloudinit_disk.node[each.key].id network_interface { - network_id = libvirt_network.k3s.id + network_id = libvirt_network.k3s.id + # false because this network runs no DHCP (see libvirt_network.k3s) — + # the node gets its address from cloud-init's network_config instead, + # so waiting on a DHCP lease here would just block forever. wait_for_lease = false } diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example index 38071b0..b9d413b 100644 --- a/terraform/terraform.tfvars.example +++ b/terraform/terraform.tfvars.example @@ -2,8 +2,8 @@ # never commit real secrets. # Default (qemu:///system) assumes you're running tofu on the T630 itself -# as the 'k8s' user - see docs/SETUP.md step 2. Leave commented out unless -# you're running Terraform from a separate workstation instead. +# as the 'k8s' user - see docs/01-bootstrap.md step 2. Leave commented out +# unless you're running Terraform from a separate workstation instead. # libvirt_uri = "qemu+ssh://k8s@t630.lan/system?keyfile=/home/you/.ssh/id_ed25519" ssh_public_key = "ssh-ed25519 AAAA... you@workstation" diff --git a/terraform/variables.tf b/terraform/variables.tf index 72000ab..5a7395e 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,5 +1,5 @@ variable "libvirt_uri" { - description = "libvirt connection URI. Default assumes tofu/kubectl/flux all run directly on the T630 as the 'k8s' user (see docs/SETUP.md step 2) - simplest option, since node IPs (network_cidr) are only directly reachable from the T630 itself. Use qemu+ssh://k8s@t630.lan/system?keyfile=... instead if you'd rather run Terraform from a separate workstation (you'll then need an SSH tunnel for kubectl/flux to reach node IPs - see step 6)." + description = "libvirt connection URI. Default assumes tofu/kubectl/flux all run directly on the T630 as the 'k8s' user (see docs/01-bootstrap.md step 2) - simplest option, since node IPs (network_cidr) are only directly reachable from the T630 itself. Use qemu+ssh://k8s@t630.lan/system?keyfile=... instead if you'd rather run Terraform from a separate workstation (you'll then need an SSH tunnel for kubectl/flux to reach node IPs - see docs/04-tofu.md step 3)." type = string default = "qemu:///system" } @@ -46,7 +46,7 @@ variable "k3s_token" { } variable "k8s_api_hostname" { - description = "LAN-only hostname for the k3s API server, added to the server's TLS SAN list so client-cert kubeconfigs validate against it. Resolve it via local DNS only (never a public record) - see docs/SETUP.md step 13." + description = "LAN-only hostname for the k3s API server, added to the server's TLS SAN list so client-cert kubeconfigs validate against it. Resolve it via local DNS only (never a public record) - see docs/04-tofu.md step 5." type = string default = "k8s-api.boglabob.com" } diff --git a/terraform/versions.tf b/terraform/versions.tf index 4a2dbfc..d89f1e1 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -3,7 +3,16 @@ terraform { required_providers { libvirt = { - source = "dmacvicar/libvirt" + source = "dmacvicar/libvirt" + # Deliberately pinned to the 0.8.x line, not just "not yet upgraded." + # v0.9.0 (Nov 2025) was an intentional, permanent fork of this + # provider to a fully-regenerated schema mapping 1:1 to libvirt's own + # XML — the maintainer kept 0.8.x alive in parallel specifically for + # existing configs like this one, rather than it being a transitional + # version to move past. `~> 0.8` (>= 0.8.0, < 0.9.0) stays on that + # legacy line on purpose; every resource block in main.tf uses that + # schema (verified against the provider's own docs at tag v0.8.3 — + # see docs/04-tofu.md's "Where this track actually stands"). version = "~> 0.8" } } @@ -11,7 +20,7 @@ terraform { # qemu+ssh:// so this can be run from your workstation against the T630; # requires the connecting user to be in the T630's `libvirt` and `kvm` -# groups (see docs/SETUP.md step 2) — no sudo needed after that. +# groups (see docs/01-bootstrap.md step 2) — no sudo needed after that. provider "libvirt" { uri = var.libvirt_uri }