Makefile

@@ -1,4 +1,4 @@
-.PHONY: init plan apply destroy clean kubeconfig talosconfig argocd-password bootstrap decrypt post-apply
+.PHONY: init plan apply destroy clean kubeconfig talosconfig argocd-password bootstrap decrypt post-apply fluentbit-secret
 
 SECRET ?= dumpnet
 TF_DIR=terraform
@@ -13,8 +13,6 @@ plan:
 
 apply:
 	cd $(TF_DIR) && terraform apply -target=talos_cluster_kubeconfig.this
-	@echo "==> Waiting 60s for Kubernetes API to be ready..."
-	@sleep 60
 	cd $(TF_DIR) && terraform apply
 
 destroy:
@@ -77,6 +75,20 @@ decrypt:
 	done
 	@echo "Decrypted files are in /tmp/ - they will not persist after reboot"
 
+# Create fluent-bit AWS credentials secret in cluster
+fluentbit-secret:
+	@KEY_ID=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \
+	  --query SecretString --output text | python3 -c \
+	  "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_access_key_id'])") && \
+	SECRET_KEY=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \
+	  --query SecretString --output text | python3 -c \
+	  "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_secret_access_key'])") && \
+	kubectl create secret generic fluentbit-aws-credentials \
+	  --namespace fluent-bit \
+	  --from-literal=AWS_ACCESS_KEY_ID=$$KEY_ID \
+	  --from-literal=AWS_SECRET_ACCESS_KEY=$$SECRET_KEY \
+	  --dry-run=client -o yaml | kubectl apply -f -
+
 # Run all post-apply steps (run once after fresh cluster creation)
 post-apply:
 	scripts/post-apply.sh

charts/fluent-bit/Chart.yaml

@@ -1,5 +0,0 @@
-apiVersion: v2
-name: fluent-bit-namespace
-description: fluent-bit namespace with privileged PodSecurity
-type: application
-version: 0.1.0

charts/fluent-bit/templates/namespace.yaml

@@ -1,8 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: fluent-bit
-  labels:
-    pod-security.kubernetes.io/enforce: privileged
-    pod-security.kubernetes.io/audit: privileged
-    pod-security.kubernetes.io/warn: privileged

charts/fluent-bit/values.yaml

@@ -5,6 +5,18 @@
 
 kind: DaemonSet
 
+env:
+  - name: AWS_ACCESS_KEY_ID
+    valueFrom:
+      secretKeyRef:
+        name: fluentbit-aws-credentials
+        key: AWS_ACCESS_KEY_ID
+  - name: AWS_SECRET_ACCESS_KEY
+    valueFrom:
+      secretKeyRef:
+        name: fluentbit-aws-credentials
+        key: AWS_SECRET_ACCESS_KEY
+
 config:
   service: |
     [SERVICE]

charts/ingress-nginx/Chart.yaml

@@ -1,5 +0,0 @@
-apiVersion: v2
-name: ingress-nginx-namespace
-description: ingress-nginx namespace with privileged PodSecurity
-type: application
-version: 0.1.0

manifests/cluster/fluent-bit.yaml

@@ -1,26 +1,5 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
-metadata:
-  name: fluent-bit-namespace
-  namespace: argocd
-spec:
-  project: default
-  source:
-    repoURL: https://forge.keane.sh/ian/dumpnet-argo.git
-    targetRevision: HEAD
-    path: charts/fluent-bit
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: fluent-bit
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
 metadata:
   name: fluent-bit
   namespace: argocd
@@ -45,4 +24,4 @@ spec:
       prune: true
       selfHeal: true
     syncOptions:
-      - CreateNamespace=false
+      - CreateNamespace=true

manifests/cluster/ingress-nginx.yaml

@@ -1,26 +1,5 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
-metadata:
-  name: ingress-nginx-namespace
-  namespace: argocd
-spec:
-  project: default
-  source:
-    repoURL: https://forge.keane.sh/ian/dumpnet-argo.git
-    targetRevision: HEAD
-    path: charts/ingress-nginx
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: ingress-nginx
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
 metadata:
   name: ingress-nginx
   namespace: argocd

terraform/argocd.tf

@@ -19,6 +19,10 @@ resource "aws_secretsmanager_secret_version" "dumpnet" {
       talosconfig           = data.talos_client_configuration.this.talos_config
       kubeconfig            = talos_cluster_kubeconfig.this.kubeconfig_raw
     }
+    fluentbit = {
+      aws_access_key_id     = aws_iam_access_key.fluentbit.id
+      aws_secret_access_key = aws_iam_access_key.fluentbit.secret
+    }
   })
 }
 

terraform/iam.tf

@@ -1,41 +0,0 @@
-# IAM role for the EC2 node
-# Grants the node (and all pods on it) access to AWS services via instance metadata.
-# Add policies here as new services need AWS access.
-
-resource "aws_iam_role" "node" {
-  name = "${var.cluster_name}-node"
-
-  assume_role_policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [{
-      Effect    = "Allow"
-      Principal = { Service = "ec2.amazonaws.com" }
-      Action    = "sts:AssumeRole"
-    }]
-  })
-}
-
-resource "aws_iam_instance_profile" "node" {
-  name = "${var.cluster_name}-node"
-  role = aws_iam_role.node.name
-}
-
-# S3 access for fluent-bit log shipping
-resource "aws_iam_role_policy" "node_s3_logs" {
-  name = "s3-logs-write"
-  role = aws_iam_role.node.id
-
-  policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [
-      {
-        Effect   = "Allow"
-        Action   = ["s3:PutObject", "s3:GetObject", "s3:ListBucket"]
-        Resource = [
-          aws_s3_bucket.logs.arn,
-          "${aws_s3_bucket.logs.arn}/*"
-        ]
-      }
-    ]
-  })
-}

terraform/logging.tf

@@ -60,6 +60,31 @@ resource "aws_athena_workgroup" "logs" {
   }
 }
 
+# IAM user for fluent-bit to write to S3
+resource "aws_iam_user" "fluentbit" {
+  name = "${var.cluster_name}-fluentbit"
+}
+
+resource "aws_iam_access_key" "fluentbit" {
+  user = aws_iam_user.fluentbit.name
+}
+
+resource "aws_iam_user_policy" "fluentbit" {
+  name = "fluentbit-s3-write"
+  user = aws_iam_user.fluentbit.name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = ["s3:PutObject"]
+        Resource = "${aws_s3_bucket.logs.arn}/*"
+      }
+    ]
+  })
+}
+
 output "logs_bucket" {
   value = aws_s3_bucket.logs.bucket
 }

terraform/main.tf

@@ -66,7 +66,6 @@ resource "aws_eip_association" "controlplane" {
 resource "aws_instance" "controlplane" {
   ami           = var.ami_id
   instance_type = var.instance_type
-  iam_instance_profile = aws_iam_instance_profile.node.name
 
   network_interface {
     network_interface_id = aws_network_interface.controlplane.id

terraform/namespaces.tf

@@ -1,5 +1,19 @@
-# Namespaces that need privileged PodSecurity are managed as Helm charts
+provider "kubernetes" {
-# in charts/<name>/templates/namespace.yaml so ArgoCD creates them with
+  host                   = "https://${aws_eip.controlplane.public_ip}:6443"
-# the correct labels before deploying workloads into them.
+  cluster_ca_certificate = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate)
-#
+  client_certificate     = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.client_certificate)
-# This file is intentionally empty.
+  client_key             = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.client_key)
+}
+
+resource "kubernetes_namespace" "ingress_nginx" {
+  metadata {
+    name = "ingress-nginx"
+    labels = {
+      "pod-security.kubernetes.io/enforce" = "privileged"
+      "pod-security.kubernetes.io/audit"   = "privileged"
+      "pod-security.kubernetes.io/warn"    = "privileged"
+    }
+  }
+
+  depends_on = [talos_cluster_kubeconfig.this]
+}