Move namespaces out of terraform

Makefile

@@ -1,4 +1,4 @@
-.PHONY: init plan apply destroy clean kubeconfig talosconfig argocd-password bootstrap decrypt post-apply fluentbit-secret
+.PHONY: init plan apply destroy clean kubeconfig talosconfig argocd-password bootstrap decrypt post-apply
 
 SECRET ?= dumpnet
 TF_DIR=terraform
@@ -13,6 +13,8 @@ plan:
 
 apply:
 	cd $(TF_DIR) && terraform apply -target=talos_cluster_kubeconfig.this
+	@echo "==> Waiting 60s for Kubernetes API to be ready..."
+	@sleep 60
 	cd $(TF_DIR) && terraform apply
 
 destroy:
@@ -75,20 +77,6 @@ decrypt:
 	done
 	@echo "Decrypted files are in /tmp/ - they will not persist after reboot"
 
-# Create fluent-bit AWS credentials secret in cluster
-fluentbit-secret:
-	@KEY_ID=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \
-	  --query SecretString --output text | python3 -c \
-	  "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_access_key_id'])") && \
-	SECRET_KEY=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \
-	  --query SecretString --output text | python3 -c \
-	  "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_secret_access_key'])") && \
-	kubectl create secret generic fluentbit-aws-credentials \
-	  --namespace fluent-bit \
-	  --from-literal=AWS_ACCESS_KEY_ID=$$KEY_ID \
-	  --from-literal=AWS_SECRET_ACCESS_KEY=$$SECRET_KEY \
-	  --dry-run=client -o yaml | kubectl apply -f -
-
 # Run all post-apply steps (run once after fresh cluster creation)
 post-apply:
 	scripts/post-apply.sh

charts/fluent-bit/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: fluent-bit-namespace
+description: fluent-bit namespace with privileged PodSecurity
+type: application
+version: 0.1.0

charts/fluent-bit/templates/namespace.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: fluent-bit
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged

charts/fluent-bit/values.yaml

@@ -5,18 +5,6 @@
 
 kind: DaemonSet
 
-env:
-  - name: AWS_ACCESS_KEY_ID
-    valueFrom:
-      secretKeyRef:
-        name: fluentbit-aws-credentials
-        key: AWS_ACCESS_KEY_ID
-  - name: AWS_SECRET_ACCESS_KEY
-    valueFrom:
-      secretKeyRef:
-        name: fluentbit-aws-credentials
-        key: AWS_SECRET_ACCESS_KEY
-
 config:
   service: |
     [SERVICE]

charts/ingress-nginx/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: ingress-nginx-namespace
+description: ingress-nginx namespace with privileged PodSecurity
+type: application
+version: 0.1.0

manifests/cluster/fluent-bit.yaml

@@ -1,5 +1,26 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
+metadata:
+  name: fluent-bit-namespace
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://forge.keane.sh/ian/dumpnet-argo.git
+    targetRevision: HEAD
+    path: charts/fluent-bit
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: fluent-bit
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
 metadata:
   name: fluent-bit
   namespace: argocd
@@ -24,4 +45,4 @@ spec:
       prune: true
       selfHeal: true
     syncOptions:
-      - CreateNamespace=true
+      - CreateNamespace=false

manifests/cluster/ingress-nginx.yaml

@@ -1,5 +1,26 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
+metadata:
+  name: ingress-nginx-namespace
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://forge.keane.sh/ian/dumpnet-argo.git
+    targetRevision: HEAD
+    path: charts/ingress-nginx
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: ingress-nginx
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
 metadata:
   name: ingress-nginx
   namespace: argocd

terraform/argocd.tf

@@ -19,10 +19,6 @@ resource "aws_secretsmanager_secret_version" "dumpnet" {
       talosconfig           = data.talos_client_configuration.this.talos_config
       kubeconfig            = talos_cluster_kubeconfig.this.kubeconfig_raw
     }
-    fluentbit = {
-      aws_access_key_id     = aws_iam_access_key.fluentbit.id
-      aws_secret_access_key = aws_iam_access_key.fluentbit.secret
-    }
   })
 }
 

terraform/iam.tf

@@ -0,0 +1,41 @@
+# IAM role for the EC2 node
+# Grants the node (and all pods on it) access to AWS services via instance metadata.
+# Add policies here as new services need AWS access.
+
+resource "aws_iam_role" "node" {
+  name = "${var.cluster_name}-node"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Principal = { Service = "ec2.amazonaws.com" }
+      Action    = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_instance_profile" "node" {
+  name = "${var.cluster_name}-node"
+  role = aws_iam_role.node.name
+}
+
+# S3 access for fluent-bit log shipping
+resource "aws_iam_role_policy" "node_s3_logs" {
+  name = "s3-logs-write"
+  role = aws_iam_role.node.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = ["s3:PutObject", "s3:GetObject", "s3:ListBucket"]
+        Resource = [
+          aws_s3_bucket.logs.arn,
+          "${aws_s3_bucket.logs.arn}/*"
+        ]
+      }
+    ]
+  })
+}

terraform/logging.tf

@@ -60,31 +60,6 @@ resource "aws_athena_workgroup" "logs" {
   }
 }
 
-# IAM user for fluent-bit to write to S3
-resource "aws_iam_user" "fluentbit" {
-  name = "${var.cluster_name}-fluentbit"
-}
-
-resource "aws_iam_access_key" "fluentbit" {
-  user = aws_iam_user.fluentbit.name
-}
-
-resource "aws_iam_user_policy" "fluentbit" {
-  name = "fluentbit-s3-write"
-  user = aws_iam_user.fluentbit.name
-
-  policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [
-      {
-        Effect   = "Allow"
-        Action   = ["s3:PutObject"]
-        Resource = "${aws_s3_bucket.logs.arn}/*"
-      }
-    ]
-  })
-}
-
 output "logs_bucket" {
   value = aws_s3_bucket.logs.bucket
 }

terraform/main.tf

@@ -66,6 +66,7 @@ resource "aws_eip_association" "controlplane" {
 resource "aws_instance" "controlplane" {
   ami                  = var.ami_id
   instance_type        = var.instance_type
+  iam_instance_profile = aws_iam_instance_profile.node.name
 
   network_interface {
     network_interface_id = aws_network_interface.controlplane.id

terraform/namespaces.tf

@@ -1,19 +1,5 @@
-provider "kubernetes" {
+# Namespaces that need privileged PodSecurity are managed as Helm charts
-  host                   = "https://${aws_eip.controlplane.public_ip}:6443"
+# in charts/<name>/templates/namespace.yaml so ArgoCD creates them with
-  cluster_ca_certificate = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate)
+# the correct labels before deploying workloads into them.
-  client_certificate     = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.client_certificate)
+#
-  client_key             = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.client_key)
+# This file is intentionally empty.
-}
-
-resource "kubernetes_namespace" "ingress_nginx" {
-  metadata {
-    name = "ingress-nginx"
-    labels = {
-      "pod-security.kubernetes.io/enforce" = "privileged"
-      "pod-security.kubernetes.io/audit"   = "privileged"
-      "pod-security.kubernetes.io/warn"    = "privileged"
-    }
-  }
-
-  depends_on = [talos_cluster_kubeconfig.this]
-}