danlin/infra-nix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Docker Registry configuration for k8s-server with TLS and authentication

Browse Source
This commit is contained in:
Daniel Lindenfelser
2025-12-16 10:47:15 +01:00
parent a83976c615
commit dd943ffd16
6 changed files with 310 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+111
View File
@@ -115,6 +115,117 @@ Access examples:
- Permissions: ensure `/data/daten` and `/data/daten/share` exist and are writable (`systemd-tmpfiles` creates them)
- Firewall: SMB ports 139/445 are open by module config
---
## K8s-Server: Docker Registry
The k8s-server runs a private Docker Registry v2 with self-signed TLS certificate. Authentication is handled by Caddy for external access.
**Access:**
- External URL: `https://registry.home.lindenfelser.de` (authenticated via Caddy)
- Internal URL: `https://10.202.82.7:5000` (direct, no auth - K8s pods)
- Default credentials: `admin` / `changeme` (Caddy basic auth)
- Storage: `/var/lib/docker-registry` (root partition)
### 1) Login from external machine
```bash
# Login to registry
docker login registry.home.lindenfelser.de
# Username: admin
# Password: changeme
```
### 2) Push an image
```bash
# Tag your image
docker tag myapp:latest registry.home.lindenfelser.de/myapp:latest
# Push to registry
docker push registry.home.lindenfelser.de/myapp:latest
```
### 3) Pull from Kubernetes pods
The k8s cluster is configured to authenticate automatically. Create a deployment:
```yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: myapp
spec:
replicas: 1
selector:
matchLabels:
app: myapp
template:
metadata:
labels:
app: myapp
spec:
containers:
- name: myapp
image: registry.home.lindenfelser.de/myapp:latest
```
### 4) Change registry password
Authentication is handled by Caddy. Generate new password hash and update:
```bash
# Generate new password hash locally
caddy hash-password --plaintext 'yournewpassword'
```
Then update the hash in [modules/gateway.nix](modules/gateway.nix) in the `basicauth` section and redeploy:
```bash
make deploy-gateway
```
### 5) Monitor storage usage
Check daily storage logs:
```bash
# View growth tracking
sudo journalctl -u docker-registry-growth-tracker
# View last 30 days of size tracking
sudo tail -n 30 /var/log/docker-registry-growth.log
# Check current usage
sudo du -sh /var/lib/docker-registry
df -h /
```
### 6) Manual garbage collection
Garbage collection runs automatically every Sunday at 03:00. To run manually:
```bash
sudo systemctl start docker-registry-garbage-collect
sudo journalctl -u docker-registry-garbage-collect -e
```
### 7) List images in registry
```bash
# List all repositories
curl -u admin:changeme https://registry.home.lindenfelser.de/v2/_catalog
# List tags for a specific image
curl -u admin:changeme https://registry.home.lindenfelser.de/v2/myapp/tags/list
```
### Troubleshooting
- **TLS certificate errors**: Registry uses self-signed certificate. External Docker clients need to add to insecure registries or install the cert
- **Authentication fails**: Verify Caddy basicauth configuration in [modules/gateway.nix](modules/gateway.nix). K8s pods access registry directly without auth.
- **Storage full**: Check root partition usage with `df -h /` and run garbage collection
- **K8s pods can't pull**: Verify `registries.yaml` points to internal registry (10.202.82.7:5000) and restart k3s: `sudo systemctl restart k3s`
````
---
hosts/k8s-server/configuration.nix
+1
View File
@@ -5,6 +5,7 @@
./hardware-configuration.nix
../../modules/common.nix
../../modules/kubernetes.nix
../../modules/docker-registry.nix
];
networking.hostName = "k8s-server";
modules/common.nix
+1
View File
@@ -39,6 +39,7 @@
htop
curl
wget
apacheHttpd # Provides htpasswd for registry password management
];
# QEMU guest agent (for VM integration when running as guest)
modules/docker-registry.nix
+164
View File
@@ -0,0 +1,164 @@
{ config, pkgs, ... }:
{
########################################
# Docker Registry v2
########################################
services.dockerRegistry = {
enable = true;
port = 5000;
listenAddress = "0.0.0.0";
# Enable image deletion and garbage collection
enableDelete = true;
enableGarbageCollect = true;
garbageCollectDates = "Sun 03:00";
# Storage location (root partition)
storagePath = "/var/lib/docker-registry";
# TLS configuration (for internal access)
extraConfig = {
http = {
tls = {
certificate = "/var/lib/docker-registry/certs/registry.crt";
key = "/var/lib/docker-registry/certs/registry.key";
};
};
};
};
########################################
# Firewall
########################################
networking.firewall.allowedTCPPorts = [ 5000 ];
########################################
# Setup: directories, certificates, htpasswd
########################################
systemd.tmpfiles.rules = [
"d /var/lib/docker-registry 0755 root root -"
"d /var/lib/docker-registry/certs 0755 root root -"
];
# Generate self-signed certificate
systemd.services.docker-registry-setup = {
description = "Docker Registry initial setup";
wantedBy = [ "multi-user.target" ];
before = [ "docker-registry.service" ];
path = with pkgs; [ openssl apacheHttpd ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
# Create self-signed certificate if it doesn't exist
if [ ! -f /var/lib/docker-registry/certs/registry.crt ]; then
echo "Generating self-signed certificate for Docker Registry..."
${pkgs.openssl}/bin/openssl req -x509 -newkey rsa:4096 -nodes \
-keyout /var/lib/docker-registry/certs/registry.key \
-out /var/lib/docker-registry/certs/registry.crt \
-days 3650 \
-subj "/CN=registry.home.lindenfelser.de" \
-addext "subjectAltName=DNS:registry.home.lindenfelser.de,DNS:k8s-server,IP:10.202.82.7"
chmod 644 /var/lib/docker-registry/certs/registry.key
chmod 644 /var/lib/docker-registry/certs/registry.crt
echo "Certificate generated successfully"
fi
# Ensure correct ownership
chown -R docker-registry:docker-registry /var/lib/docker-registry
'';
};
########################################
# Storage Monitoring (80% threshold)
########################################
systemd.services.docker-registry-storage-check = {
description = "Check Docker Registry storage usage";
path = with pkgs; [ coreutils util-linux ];
serviceConfig = {
Type = "oneshot";
User = "root";
};
script = ''
REGISTRY_PATH="/var/lib/docker-registry"
THRESHOLD=80
if [ ! -d "$REGISTRY_PATH" ]; then
echo "Registry path does not exist yet"
exit 0
fi
# Get disk usage percentage of the filesystem containing the registry
USAGE=$(df -h "$REGISTRY_PATH" | awk 'NR==2 {print $5}' | sed 's/%//')
REGISTRY_SIZE=$(du -sh "$REGISTRY_PATH" | cut -f1)
echo "Docker Registry storage: $REGISTRY_SIZE (filesystem usage: $USAGE%)"
if [ "$USAGE" -gt "$THRESHOLD" ]; then
echo "WARNING: Filesystem usage ($USAGE%) exceeds threshold ($THRESHOLD%)"
echo "Consider cleaning up old images or expanding storage"
fi
'';
};
systemd.timers.docker-registry-storage-check = {
description = "Timer for Docker Registry storage check";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
};
};
########################################
# Daily Storage Growth Tracking
########################################
systemd.services.docker-registry-growth-tracker = {
description = "Track Docker Registry storage growth";
path = with pkgs; [ coreutils ];
serviceConfig = {
Type = "oneshot";
User = "root";
};
script = ''
REGISTRY_PATH="/var/lib/docker-registry"
LOG_FILE="/var/log/docker-registry-growth.log"
if [ ! -d "$REGISTRY_PATH" ]; then
echo "Registry path does not exist yet"
exit 0
fi
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
SIZE_BYTES=$(du -sb "$REGISTRY_PATH" | cut -f1)
SIZE_HUMAN=$(du -sh "$REGISTRY_PATH" | cut -f1)
echo "$TIMESTAMP | Size: $SIZE_HUMAN ($SIZE_BYTES bytes)" >> "$LOG_FILE"
# Keep only last 90 days of logs
if [ -f "$LOG_FILE" ]; then
tail -n 90 "$LOG_FILE" > "$LOG_FILE.tmp"
mv "$LOG_FILE.tmp" "$LOG_FILE"
fi
'';
};
systemd.timers.docker-registry-growth-tracker = {
description = "Timer for Docker Registry growth tracking";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
};
};
}
modules/gateway.nix
+14
View File
@@ -42,6 +42,20 @@
reverse_proxy 10.202.82.6:3000
'';
};
"registry.home.lindenfelser.de" = {
extraConfig = ''
@registry host registry.home.lindenfelser.de
basicauth @registry {
admin $2a$14$Ga5BCiHvtlfRjdnlI9bhseFnNZ8dwXsLz4t1FdSemA1mAUV/vA1oi
}
reverse_proxy @registry https://10.202.82.7:5000 {
transport http {
tls
tls_insecure_skip_verify
}
}
'';
};
};
};
modules/kubernetes.nix
+19
View File
@@ -12,6 +12,7 @@
"--cluster-init"
"--disable=traefik"
"--flannel-backend=vxlan"
"--tls-san=k8s.home.lindenfelser.de"
];
};
@@ -22,9 +23,27 @@
networking.firewall.allowedTCPPorts = [
6443 # Kubernetes API
10250 # Kubelet metrics
5000 # Docker Registry
];
networking.firewall.allowedUDPPorts = [
8472 # flannel VXLAN
];
########################################
# Containerd registry configuration
########################################
environment.etc."rancher/k3s/registries.yaml" = {
text = ''
mirrors:
registry.home.lindenfelser.de:
endpoint:
- "https://10.202.82.7:5000"
configs:
"10.202.82.7:5000":
tls:
insecure_skip_verify: true
'';
mode = "0644";
};
}