Refactor services heirarchically, add logging

Makefile

@@ -1,6 +1,6 @@
-.PHONY: init plan apply destroy kubeconfig talosconfig argocd-password bootstrap decrypt post-apply
+.PHONY: init plan apply destroy kubeconfig talosconfig argocd-password bootstrap decrypt post-apply fluentbit-secret
 
-SECRET=dumpnet/cluster
+SECRET=dumpnet
 TF_DIR=terraform
 SOPS_FILES=controlplane.yaml worker.yaml talosconfig
 
@@ -25,7 +25,7 @@ bootstrap:
 talosconfig:
 	aws secretsmanager get-secret-value --secret-id $(SECRET) \
 	  --query SecretString --output text | python3 -c \
-	  "import sys,json; print(json.load(sys.stdin)['talosconfig'])" \
+	  "import sys,json; print(json.load(sys.stdin)['cluster']['talosconfig'])" \
 	  > /tmp/talosconfig-dumpnet
 	talosctl config merge /tmp/talosconfig-dumpnet
 	rm /tmp/talosconfig-dumpnet
@@ -34,7 +34,7 @@ talosconfig:
 kubeconfig:
 	aws secretsmanager get-secret-value --secret-id $(SECRET) \
 	  --query SecretString --output text | python3 -c \
-	  "import sys,json; print(json.load(sys.stdin)['kubeconfig'])" \
+	  "import sys,json; print(json.load(sys.stdin)['cluster']['kubeconfig'])" \
 	  > /tmp/kubeconfig-dumpnet
 	KUBECONFIG=~/.kube/config:/tmp/kubeconfig-dumpnet kubectl config view --flatten > /tmp/merged
 	mv /tmp/merged ~/.kube/config
@@ -44,7 +44,7 @@ kubeconfig:
 argocd-password:
 	@aws secretsmanager get-secret-value --secret-id $(SECRET) \
 	  --query SecretString --output text | python3 -c \
-	  "import sys,json; print(json.load(sys.stdin)['argocd_admin_password'])"
+	  "import sys,json; print(json.load(sys.stdin)['cluster']['argocd_admin_password'])"
 
 # Decrypt sensitive files to /tmp for one-off talosctl use
 # Files are never decrypted in the repo directory
@@ -57,6 +57,20 @@ decrypt:
 	done
 	@echo "Decrypted files are in /tmp/ - they will not persist after reboot"
 
+# Create fluent-bit AWS credentials secret in cluster
+fluentbit-secret:
+	@KEY_ID=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \
+	  --query SecretString --output text | python3 -c \
+	  "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_access_key_id'])") && \
+	SECRET_KEY=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \
+	  --query SecretString --output text | python3 -c \
+	  "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_secret_access_key'])") && \
+	kubectl create secret generic fluentbit-aws-credentials \
+	  --namespace fluent-bit \
+	  --from-literal=AWS_ACCESS_KEY_ID=$$KEY_ID \
+	  --from-literal=AWS_SECRET_ACCESS_KEY=$$SECRET_KEY \
+	  --dry-run=client -o yaml | kubectl apply -f -
+
 # Run all post-apply steps (run once after fresh cluster creation)
 post-apply:
 	scripts/post-apply.sh

apps/apps.yaml

@@ -1,14 +1,14 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: apps
+  name: cluster
   namespace: argocd
 spec:
   project: default
   source:
     repoURL: https://forge.keane.sh/ian/dumpnet-argo.git
     targetRevision: HEAD
-    path: manifests
+    path: manifests/cluster
   destination:
     server: https://kubernetes.default.svc
     namespace: argocd
@@ -16,3 +16,23 @@ spec:
     automated:
       prune: true
       selfHeal: true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: services
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://forge.keane.sh/ian/dumpnet-argo.git
+    targetRevision: HEAD
+    path: manifests/services
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+

charts/fluent-bit/values.yaml

@@ -0,0 +1,56 @@
+## Fluent Bit Helm values
+## AWS credentials are expected in a secret: fluent-bit/fluentbit-aws-credentials
+## with keys: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY
+## Create with: make fluentbit-secret
+
+kind: DaemonSet
+
+env:
+  - name: AWS_ACCESS_KEY_ID
+    valueFrom:
+      secretKeyRef:
+        name: fluentbit-aws-credentials
+        key: AWS_ACCESS_KEY_ID
+  - name: AWS_SECRET_ACCESS_KEY
+    valueFrom:
+      secretKeyRef:
+        name: fluentbit-aws-credentials
+        key: AWS_SECRET_ACCESS_KEY
+
+config:
+  service: |
+    [SERVICE]
+        Flush         5
+        Daemon        off
+        Log_Level     warn
+        Parsers_File  parsers.conf
+
+  inputs: |
+    [INPUT]
+        Name              tail
+        Tag               kube.*
+        Path              /var/log/containers/*.log
+        Parser            cri
+        Mem_Buf_Limit     5MB
+        Skip_Long_Lines   On
+
+  filters: |
+    [FILTER]
+        Name                kubernetes
+        Match               kube.*
+        Merge_Log           On
+        Keep_Log            Off
+        K8S-Logging.Parser  On
+        K8S-Logging.Exclude On
+
+  outputs: |
+    [OUTPUT]
+        Name                         s3
+        Match                        kube.*
+        bucket                       dumpnet-logs
+        region                       us-east-1
+        s3_key_format                /logs/%Y/%m/%d/$TAG[4].%H%M%S.gz
+        total_file_size              100M
+        upload_timeout               10m
+        compression                  gzip
+        use_put_object               On

manifests/cluster/fluent-bit.yaml

@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: fluent-bit
+  namespace: argocd
+spec:
+  project: default
+  sources:
+    - repoURL: https://forge.keane.sh/ian/dumpnet-argo.git
+      targetRevision: HEAD
+      ref: values
+    - repoURL: https://fluent.github.io/helm-charts
+      chart: fluent-bit
+      targetRevision: "*"
+      helm:
+        valueFiles:
+          - $values/charts/fluent-bit/values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: fluent-bit
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

terraform/argocd.tf

@@ -5,18 +5,24 @@ resource "random_password" "argocd_admin" {
   override_special = "!#$%&*()-_=+[]{}?"
 }
 
-# Store secrets in AWS Secrets Manager
+# Single secrets manager secret for all dumpnet secrets
 resource "aws_secretsmanager_secret" "dumpnet" {
-  name        = "dumpnet/cluster"
+  name        = "dumpnet"
-  description = "Dumpnet cluster secrets"
+  description = "All dumpnet secrets"
 }
 
 resource "aws_secretsmanager_secret_version" "dumpnet" {
   secret_id = aws_secretsmanager_secret.dumpnet.id
   secret_string = jsonencode({
+    cluster = {
       argocd_admin_password = random_password.argocd_admin.result
       talosconfig           = data.talos_client_configuration.this.talos_config
       kubeconfig            = talos_cluster_kubeconfig.this.kubeconfig_raw
+    }
+    fluentbit = {
+      aws_access_key_id     = aws_iam_access_key.fluentbit.id
+      aws_secret_access_key = aws_iam_access_key.fluentbit.secret
+    }
   })
 }
 

terraform/logging.tf

@@ -0,0 +1,86 @@
+# S3 bucket for logs
+resource "aws_s3_bucket" "logs" {
+  bucket = "dumpnet-logs"
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "logs" {
+  bucket = aws_s3_bucket.logs.id
+
+  rule {
+    id     = "expire-old-logs"
+    status = "Enabled"
+
+    filter {}
+
+    transition {
+      days          = 30
+      storage_class = "INTELLIGENT_TIERING"
+    }
+
+    expiration {
+      days = 365
+    }
+  }
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "logs" {
+  bucket = aws_s3_bucket.logs.id
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+# Athena database + workgroup
+resource "aws_athena_database" "logs" {
+  name   = "dumpnet_logs"
+  bucket = aws_s3_bucket.athena_results.bucket
+}
+
+resource "aws_s3_bucket" "athena_results" {
+  bucket = "dumpnet-athena-results"
+}
+
+resource "aws_athena_workgroup" "logs" {
+  name = "dumpnet-logs"
+
+  configuration {
+    result_configuration {
+      output_location = "s3://${aws_s3_bucket.athena_results.bucket}/results/"
+
+      encryption_configuration {
+        encryption_option = "SSE_S3"
+      }
+    }
+  }
+}
+
+# IAM user for fluent-bit to write to S3
+resource "aws_iam_user" "fluentbit" {
+  name = "dumpnet-fluentbit"
+}
+
+resource "aws_iam_access_key" "fluentbit" {
+  user = aws_iam_user.fluentbit.name
+}
+
+resource "aws_iam_user_policy" "fluentbit" {
+  name = "fluentbit-s3-write"
+  user = aws_iam_user.fluentbit.name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = ["s3:PutObject"]
+        Resource = "${aws_s3_bucket.logs.arn}/*"
+      }
+    ]
+  })
+}
+
+output "logs_bucket" {
+  value = aws_s3_bucket.logs.bucket
+}