From dd943ffd169048ea0f5d7850c61d16a6c9d0df05 Mon Sep 17 00:00:00 2001 From: Daniel Lindenfelser Date: Tue, 16 Dec 2025 10:47:15 +0100 Subject: [PATCH] Add Docker Registry configuration for k8s-server with TLS and authentication --- README.md | 111 +++++++++++++++++++ hosts/k8s-server/configuration.nix | 1 + modules/common.nix | 1 + modules/docker-registry.nix | 164 +++++++++++++++++++++++++++++ modules/gateway.nix | 14 +++ modules/kubernetes.nix | 19 ++++ 6 files changed, 310 insertions(+) create mode 100644 modules/docker-registry.nix diff --git a/README.md b/README.md index 4a92a18..12e5893 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,117 @@ Access examples: - Permissions: ensure `/data/daten` and `/data/daten/share` exist and are writable (`systemd-tmpfiles` creates them) - Firewall: SMB ports 139/445 are open by module config +--- + +## K8s-Server: Docker Registry + +The k8s-server runs a private Docker Registry v2 with self-signed TLS certificate. Authentication is handled by Caddy for external access. + +**Access:** +- External URL: `https://registry.home.lindenfelser.de` (authenticated via Caddy) +- Internal URL: `https://10.202.82.7:5000` (direct, no auth - K8s pods) +- Default credentials: `admin` / `changeme` (Caddy basic auth) +- Storage: `/var/lib/docker-registry` (root partition) + +### 1) Login from external machine + +```bash +# Login to registry +docker login registry.home.lindenfelser.de +# Username: admin +# Password: changeme +``` + +### 2) Push an image + +```bash +# Tag your image +docker tag myapp:latest registry.home.lindenfelser.de/myapp:latest + +# Push to registry +docker push registry.home.lindenfelser.de/myapp:latest +``` + +### 3) Pull from Kubernetes pods + +The k8s cluster is configured to authenticate automatically. Create a deployment: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: myapp +spec: + replicas: 1 + selector: + matchLabels: + app: myapp + template: + metadata: + labels: + app: myapp + spec: + containers: + - name: myapp + image: registry.home.lindenfelser.de/myapp:latest +``` + +### 4) Change registry password + +Authentication is handled by Caddy. Generate new password hash and update: + +```bash +# Generate new password hash locally +caddy hash-password --plaintext 'yournewpassword' +``` + +Then update the hash in [modules/gateway.nix](modules/gateway.nix) in the `basicauth` section and redeploy: + +```bash +make deploy-gateway +``` + +### 5) Monitor storage usage + +Check daily storage logs: + +```bash +# View growth tracking +sudo journalctl -u docker-registry-growth-tracker + +# View last 30 days of size tracking +sudo tail -n 30 /var/log/docker-registry-growth.log + +# Check current usage +sudo du -sh /var/lib/docker-registry +df -h / +``` + +### 6) Manual garbage collection + +Garbage collection runs automatically every Sunday at 03:00. To run manually: + +```bash +sudo systemctl start docker-registry-garbage-collect +sudo journalctl -u docker-registry-garbage-collect -e +``` + +### 7) List images in registry + +```bash +# List all repositories +curl -u admin:changeme https://registry.home.lindenfelser.de/v2/_catalog + +# List tags for a specific image +curl -u admin:changeme https://registry.home.lindenfelser.de/v2/myapp/tags/list +``` + +### Troubleshooting +- **TLS certificate errors**: Registry uses self-signed certificate. External Docker clients need to add to insecure registries or install the cert +- **Authentication fails**: Verify Caddy basicauth configuration in [modules/gateway.nix](modules/gateway.nix). K8s pods access registry directly without auth. +- **Storage full**: Check root partition usage with `df -h /` and run garbage collection +- **K8s pods can't pull**: Verify `registries.yaml` points to internal registry (10.202.82.7:5000) and restart k3s: `sudo systemctl restart k3s` + ```` --- diff --git a/hosts/k8s-server/configuration.nix b/hosts/k8s-server/configuration.nix index 46998e5..dd98034 100644 --- a/hosts/k8s-server/configuration.nix +++ b/hosts/k8s-server/configuration.nix @@ -5,6 +5,7 @@ ./hardware-configuration.nix ../../modules/common.nix ../../modules/kubernetes.nix + ../../modules/docker-registry.nix ]; networking.hostName = "k8s-server"; diff --git a/modules/common.nix b/modules/common.nix index 0ebfb9e..c39cace 100644 --- a/modules/common.nix +++ b/modules/common.nix @@ -39,6 +39,7 @@ htop curl wget + apacheHttpd # Provides htpasswd for registry password management ]; # QEMU guest agent (for VM integration when running as guest) diff --git a/modules/docker-registry.nix b/modules/docker-registry.nix new file mode 100644 index 0000000..78de624 --- /dev/null +++ b/modules/docker-registry.nix @@ -0,0 +1,164 @@ +{ config, pkgs, ... }: + +{ + ######################################## + # Docker Registry v2 + ######################################## + services.dockerRegistry = { + enable = true; + port = 5000; + listenAddress = "0.0.0.0"; + + # Enable image deletion and garbage collection + enableDelete = true; + enableGarbageCollect = true; + garbageCollectDates = "Sun 03:00"; + + # Storage location (root partition) + storagePath = "/var/lib/docker-registry"; + + # TLS configuration (for internal access) + extraConfig = { + http = { + tls = { + certificate = "/var/lib/docker-registry/certs/registry.crt"; + key = "/var/lib/docker-registry/certs/registry.key"; + }; + }; + }; + }; + + ######################################## + # Firewall + ######################################## + networking.firewall.allowedTCPPorts = [ 5000 ]; + + ######################################## + # Setup: directories, certificates, htpasswd + ######################################## + systemd.tmpfiles.rules = [ + "d /var/lib/docker-registry 0755 root root -" + "d /var/lib/docker-registry/certs 0755 root root -" + ]; + + # Generate self-signed certificate + systemd.services.docker-registry-setup = { + description = "Docker Registry initial setup"; + wantedBy = [ "multi-user.target" ]; + before = [ "docker-registry.service" ]; + path = with pkgs; [ openssl apacheHttpd ]; + + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + + script = '' + # Create self-signed certificate if it doesn't exist + if [ ! -f /var/lib/docker-registry/certs/registry.crt ]; then + echo "Generating self-signed certificate for Docker Registry..." + ${pkgs.openssl}/bin/openssl req -x509 -newkey rsa:4096 -nodes \ + -keyout /var/lib/docker-registry/certs/registry.key \ + -out /var/lib/docker-registry/certs/registry.crt \ + -days 3650 \ + -subj "/CN=registry.home.lindenfelser.de" \ + -addext "subjectAltName=DNS:registry.home.lindenfelser.de,DNS:k8s-server,IP:10.202.82.7" + chmod 644 /var/lib/docker-registry/certs/registry.key + chmod 644 /var/lib/docker-registry/certs/registry.crt + echo "Certificate generated successfully" + fi + + # Ensure correct ownership + chown -R docker-registry:docker-registry /var/lib/docker-registry + ''; + }; + + ######################################## + # Storage Monitoring (80% threshold) + ######################################## + systemd.services.docker-registry-storage-check = { + description = "Check Docker Registry storage usage"; + path = with pkgs; [ coreutils util-linux ]; + + serviceConfig = { + Type = "oneshot"; + User = "root"; + }; + + script = '' + REGISTRY_PATH="/var/lib/docker-registry" + THRESHOLD=80 + + if [ ! -d "$REGISTRY_PATH" ]; then + echo "Registry path does not exist yet" + exit 0 + fi + + # Get disk usage percentage of the filesystem containing the registry + USAGE=$(df -h "$REGISTRY_PATH" | awk 'NR==2 {print $5}' | sed 's/%//') + REGISTRY_SIZE=$(du -sh "$REGISTRY_PATH" | cut -f1) + + echo "Docker Registry storage: $REGISTRY_SIZE (filesystem usage: $USAGE%)" + + if [ "$USAGE" -gt "$THRESHOLD" ]; then + echo "WARNING: Filesystem usage ($USAGE%) exceeds threshold ($THRESHOLD%)" + echo "Consider cleaning up old images or expanding storage" + fi + ''; + }; + + systemd.timers.docker-registry-storage-check = { + description = "Timer for Docker Registry storage check"; + wantedBy = [ "timers.target" ]; + + timerConfig = { + OnCalendar = "daily"; + Persistent = true; + }; + }; + + ######################################## + # Daily Storage Growth Tracking + ######################################## + systemd.services.docker-registry-growth-tracker = { + description = "Track Docker Registry storage growth"; + path = with pkgs; [ coreutils ]; + + serviceConfig = { + Type = "oneshot"; + User = "root"; + }; + + script = '' + REGISTRY_PATH="/var/lib/docker-registry" + LOG_FILE="/var/log/docker-registry-growth.log" + + if [ ! -d "$REGISTRY_PATH" ]; then + echo "Registry path does not exist yet" + exit 0 + fi + + TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S') + SIZE_BYTES=$(du -sb "$REGISTRY_PATH" | cut -f1) + SIZE_HUMAN=$(du -sh "$REGISTRY_PATH" | cut -f1) + + echo "$TIMESTAMP | Size: $SIZE_HUMAN ($SIZE_BYTES bytes)" >> "$LOG_FILE" + + # Keep only last 90 days of logs + if [ -f "$LOG_FILE" ]; then + tail -n 90 "$LOG_FILE" > "$LOG_FILE.tmp" + mv "$LOG_FILE.tmp" "$LOG_FILE" + fi + ''; + }; + + systemd.timers.docker-registry-growth-tracker = { + description = "Timer for Docker Registry growth tracking"; + wantedBy = [ "timers.target" ]; + + timerConfig = { + OnCalendar = "daily"; + Persistent = true; + }; + }; +} diff --git a/modules/gateway.nix b/modules/gateway.nix index cd291b9..802f25a 100644 --- a/modules/gateway.nix +++ b/modules/gateway.nix @@ -42,6 +42,20 @@ reverse_proxy 10.202.82.6:3000 ''; }; + "registry.home.lindenfelser.de" = { + extraConfig = '' + @registry host registry.home.lindenfelser.de + basicauth @registry { + admin $2a$14$Ga5BCiHvtlfRjdnlI9bhseFnNZ8dwXsLz4t1FdSemA1mAUV/vA1oi + } + reverse_proxy @registry https://10.202.82.7:5000 { + transport http { + tls + tls_insecure_skip_verify + } + } + ''; + }; }; }; diff --git a/modules/kubernetes.nix b/modules/kubernetes.nix index 275e235..c52c0d7 100644 --- a/modules/kubernetes.nix +++ b/modules/kubernetes.nix @@ -12,6 +12,7 @@ "--cluster-init" "--disable=traefik" "--flannel-backend=vxlan" + "--tls-san=k8s.home.lindenfelser.de" ]; }; @@ -22,9 +23,27 @@ networking.firewall.allowedTCPPorts = [ 6443 # Kubernetes API 10250 # Kubelet metrics + 5000 # Docker Registry ]; networking.firewall.allowedUDPPorts = [ 8472 # flannel VXLAN ]; + + ######################################## + # Containerd registry configuration + ######################################## + environment.etc."rancher/k3s/registries.yaml" = { + text = '' + mirrors: + registry.home.lindenfelser.de: + endpoint: + - "https://10.202.82.7:5000" + configs: + "10.202.82.7:5000": + tls: + insecure_skip_verify: true + ''; + mode = "0644"; + }; }