From 865831f1de1c3bc79b7e9963af7160736432739e Mon Sep 17 00:00:00 2001 From: Ian Keane Date: Tue, 18 Aug 2026 13:17:40 -0400 Subject: [PATCH 1/2] Setup s3 access at ec2 level --- Makefile | 18 +++------------ charts/fluent-bit/values.yaml | 12 ---------- terraform/argocd.tf | 4 ---- terraform/iam.tf | 41 +++++++++++++++++++++++++++++++++++ terraform/logging.tf | 25 --------------------- terraform/main.tf | 5 +++-- 6 files changed, 47 insertions(+), 58 deletions(-) create mode 100644 terraform/iam.tf diff --git a/Makefile b/Makefile index c2e9eb2..cb77b2f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: init plan apply destroy clean kubeconfig talosconfig argocd-password bootstrap decrypt post-apply fluentbit-secret +.PHONY: init plan apply destroy clean kubeconfig talosconfig argocd-password bootstrap decrypt post-apply SECRET ?= dumpnet TF_DIR=terraform @@ -13,6 +13,8 @@ plan: apply: cd $(TF_DIR) && terraform apply -target=talos_cluster_kubeconfig.this + @echo "==> Waiting 60s for Kubernetes API to be ready..." + @sleep 60 cd $(TF_DIR) && terraform apply destroy: @@ -75,20 +77,6 @@ decrypt: done @echo "Decrypted files are in /tmp/ - they will not persist after reboot" -# Create fluent-bit AWS credentials secret in cluster -fluentbit-secret: - @KEY_ID=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \ - --query SecretString --output text | python3 -c \ - "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_access_key_id'])") && \ - SECRET_KEY=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \ - --query SecretString --output text | python3 -c \ - "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_secret_access_key'])") && \ - kubectl create secret generic fluentbit-aws-credentials \ - --namespace fluent-bit \ - --from-literal=AWS_ACCESS_KEY_ID=$$KEY_ID \ - --from-literal=AWS_SECRET_ACCESS_KEY=$$SECRET_KEY \ - --dry-run=client -o yaml | kubectl apply -f - - # Run all post-apply steps (run once after fresh cluster creation) post-apply: scripts/post-apply.sh diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index a6fdb53..9df1115 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -5,18 +5,6 @@ kind: DaemonSet -env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: fluentbit-aws-credentials - key: AWS_ACCESS_KEY_ID - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: fluentbit-aws-credentials - key: AWS_SECRET_ACCESS_KEY - config: service: | [SERVICE] diff --git a/terraform/argocd.tf b/terraform/argocd.tf index 7aed63f..0d6d3c1 100644 --- a/terraform/argocd.tf +++ b/terraform/argocd.tf @@ -19,10 +19,6 @@ resource "aws_secretsmanager_secret_version" "dumpnet" { talosconfig = data.talos_client_configuration.this.talos_config kubeconfig = talos_cluster_kubeconfig.this.kubeconfig_raw } - fluentbit = { - aws_access_key_id = aws_iam_access_key.fluentbit.id - aws_secret_access_key = aws_iam_access_key.fluentbit.secret - } }) } diff --git a/terraform/iam.tf b/terraform/iam.tf new file mode 100644 index 0000000..55bd744 --- /dev/null +++ b/terraform/iam.tf @@ -0,0 +1,41 @@ +# IAM role for the EC2 node +# Grants the node (and all pods on it) access to AWS services via instance metadata. +# Add policies here as new services need AWS access. + +resource "aws_iam_role" "node" { + name = "${var.cluster_name}-node" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_instance_profile" "node" { + name = "${var.cluster_name}-node" + role = aws_iam_role.node.name +} + +# S3 access for fluent-bit log shipping +resource "aws_iam_role_policy" "node_s3_logs" { + name = "s3-logs-write" + role = aws_iam_role.node.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["s3:PutObject", "s3:GetObject", "s3:ListBucket"] + Resource = [ + aws_s3_bucket.logs.arn, + "${aws_s3_bucket.logs.arn}/*" + ] + } + ] + }) +} diff --git a/terraform/logging.tf b/terraform/logging.tf index c2215db..a272f62 100644 --- a/terraform/logging.tf +++ b/terraform/logging.tf @@ -60,31 +60,6 @@ resource "aws_athena_workgroup" "logs" { } } -# IAM user for fluent-bit to write to S3 -resource "aws_iam_user" "fluentbit" { - name = "${var.cluster_name}-fluentbit" -} - -resource "aws_iam_access_key" "fluentbit" { - user = aws_iam_user.fluentbit.name -} - -resource "aws_iam_user_policy" "fluentbit" { - name = "fluentbit-s3-write" - user = aws_iam_user.fluentbit.name - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = ["s3:PutObject"] - Resource = "${aws_s3_bucket.logs.arn}/*" - } - ] - }) -} - output "logs_bucket" { value = aws_s3_bucket.logs.bucket } diff --git a/terraform/main.tf b/terraform/main.tf index b3bd8f3..400462b 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -64,8 +64,9 @@ resource "aws_eip_association" "controlplane" { # Launch control plane instance resource "aws_instance" "controlplane" { - ami = var.ami_id - instance_type = var.instance_type + ami = var.ami_id + instance_type = var.instance_type + iam_instance_profile = aws_iam_instance_profile.node.name network_interface { network_interface_id = aws_network_interface.controlplane.id From 6cecb90baf713fc8157902e0b31751e71ddf4994 Mon Sep 17 00:00:00 2001 From: Ian Keane Date: Tue, 18 Aug 2026 13:46:49 -0400 Subject: [PATCH 2/2] Move namespaces out of terraform --- charts/fluent-bit/Chart.yaml | 5 ++++ charts/fluent-bit/templates/namespace.yaml | 8 +++++++ charts/ingress-nginx/Chart.yaml | 5 ++++ .../{ => templates}/namespace.yaml | 0 manifests/cluster/fluent-bit.yaml | 23 +++++++++++++++++- manifests/cluster/ingress-nginx.yaml | 21 ++++++++++++++++ terraform/namespaces.tf | 24 ++++--------------- 7 files changed, 66 insertions(+), 20 deletions(-) create mode 100644 charts/fluent-bit/Chart.yaml create mode 100644 charts/fluent-bit/templates/namespace.yaml create mode 100644 charts/ingress-nginx/Chart.yaml rename charts/ingress-nginx/{ => templates}/namespace.yaml (100%) diff --git a/charts/fluent-bit/Chart.yaml b/charts/fluent-bit/Chart.yaml new file mode 100644 index 0000000..97d63be --- /dev/null +++ b/charts/fluent-bit/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: fluent-bit-namespace +description: fluent-bit namespace with privileged PodSecurity +type: application +version: 0.1.0 diff --git a/charts/fluent-bit/templates/namespace.yaml b/charts/fluent-bit/templates/namespace.yaml new file mode 100644 index 0000000..e69bd65 --- /dev/null +++ b/charts/fluent-bit/templates/namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: fluent-bit + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged diff --git a/charts/ingress-nginx/Chart.yaml b/charts/ingress-nginx/Chart.yaml new file mode 100644 index 0000000..b73197a --- /dev/null +++ b/charts/ingress-nginx/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: ingress-nginx-namespace +description: ingress-nginx namespace with privileged PodSecurity +type: application +version: 0.1.0 diff --git a/charts/ingress-nginx/namespace.yaml b/charts/ingress-nginx/templates/namespace.yaml similarity index 100% rename from charts/ingress-nginx/namespace.yaml rename to charts/ingress-nginx/templates/namespace.yaml diff --git a/manifests/cluster/fluent-bit.yaml b/manifests/cluster/fluent-bit.yaml index 2867276..705bd0f 100644 --- a/manifests/cluster/fluent-bit.yaml +++ b/manifests/cluster/fluent-bit.yaml @@ -1,5 +1,26 @@ apiVersion: argoproj.io/v1alpha1 kind: Application +metadata: + name: fluent-bit-namespace + namespace: argocd +spec: + project: default + source: + repoURL: https://forge.keane.sh/ian/dumpnet-argo.git + targetRevision: HEAD + path: charts/fluent-bit + destination: + server: https://kubernetes.default.svc + namespace: fluent-bit + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application metadata: name: fluent-bit namespace: argocd @@ -24,4 +45,4 @@ spec: prune: true selfHeal: true syncOptions: - - CreateNamespace=true + - CreateNamespace=false diff --git a/manifests/cluster/ingress-nginx.yaml b/manifests/cluster/ingress-nginx.yaml index 84a567c..1ba6726 100644 --- a/manifests/cluster/ingress-nginx.yaml +++ b/manifests/cluster/ingress-nginx.yaml @@ -1,5 +1,26 @@ apiVersion: argoproj.io/v1alpha1 kind: Application +metadata: + name: ingress-nginx-namespace + namespace: argocd +spec: + project: default + source: + repoURL: https://forge.keane.sh/ian/dumpnet-argo.git + targetRevision: HEAD + path: charts/ingress-nginx + destination: + server: https://kubernetes.default.svc + namespace: ingress-nginx + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application metadata: name: ingress-nginx namespace: argocd diff --git a/terraform/namespaces.tf b/terraform/namespaces.tf index 8ed4586..40f54f7 100644 --- a/terraform/namespaces.tf +++ b/terraform/namespaces.tf @@ -1,19 +1,5 @@ -provider "kubernetes" { - host = "https://${aws_eip.controlplane.public_ip}:6443" - cluster_ca_certificate = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate) - client_certificate = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.client_certificate) - client_key = base64decode(talos_cluster_kubeconfig.this.kubernetes_client_configuration.client_key) -} - -resource "kubernetes_namespace" "ingress_nginx" { - metadata { - name = "ingress-nginx" - labels = { - "pod-security.kubernetes.io/enforce" = "privileged" - "pod-security.kubernetes.io/audit" = "privileged" - "pod-security.kubernetes.io/warn" = "privileged" - } - } - - depends_on = [talos_cluster_kubeconfig.this] -} +# Namespaces that need privileged PodSecurity are managed as Helm charts +# in charts//templates/namespace.yaml so ArgoCD creates them with +# the correct labels before deploying workloads into them. +# +# This file is intentionally empty.