Refactor services heirarchically, add logging

terraform/argocd.tf

@@ -5,18 +5,24 @@ resource "random_password" "argocd_admin" {
   override_special = "!#$%&*()-_=+[]{}?"
 }
 
-# Store secrets in AWS Secrets Manager
+# Single secrets manager secret for all dumpnet secrets
 resource "aws_secretsmanager_secret" "dumpnet" {
-  name        = "dumpnet/cluster"
-  description = "Dumpnet cluster secrets"
+  name        = "dumpnet"
+  description = "All dumpnet secrets"
 }
 
 resource "aws_secretsmanager_secret_version" "dumpnet" {
   secret_id = aws_secretsmanager_secret.dumpnet.id
   secret_string = jsonencode({
-    argocd_admin_password = random_password.argocd_admin.result
-    talosconfig           = data.talos_client_configuration.this.talos_config
-    kubeconfig            = talos_cluster_kubeconfig.this.kubeconfig_raw
+    cluster = {
+      argocd_admin_password = random_password.argocd_admin.result
+      talosconfig           = data.talos_client_configuration.this.talos_config
+      kubeconfig            = talos_cluster_kubeconfig.this.kubeconfig_raw
+    }
+    fluentbit = {
+      aws_access_key_id     = aws_iam_access_key.fluentbit.id
+      aws_secret_access_key = aws_iam_access_key.fluentbit.secret
+    }
   })
 }
 

terraform/logging.tf

@@ -0,0 +1,86 @@
+# S3 bucket for logs
+resource "aws_s3_bucket" "logs" {
+  bucket = "dumpnet-logs"
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "logs" {
+  bucket = aws_s3_bucket.logs.id
+
+  rule {
+    id     = "expire-old-logs"
+    status = "Enabled"
+
+    filter {}
+
+    transition {
+      days          = 30
+      storage_class = "INTELLIGENT_TIERING"
+    }
+
+    expiration {
+      days = 365
+    }
+  }
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "logs" {
+  bucket = aws_s3_bucket.logs.id
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+# Athena database + workgroup
+resource "aws_athena_database" "logs" {
+  name   = "dumpnet_logs"
+  bucket = aws_s3_bucket.athena_results.bucket
+}
+
+resource "aws_s3_bucket" "athena_results" {
+  bucket = "dumpnet-athena-results"
+}
+
+resource "aws_athena_workgroup" "logs" {
+  name = "dumpnet-logs"
+
+  configuration {
+    result_configuration {
+      output_location = "s3://${aws_s3_bucket.athena_results.bucket}/results/"
+
+      encryption_configuration {
+        encryption_option = "SSE_S3"
+      }
+    }
+  }
+}
+
+# IAM user for fluent-bit to write to S3
+resource "aws_iam_user" "fluentbit" {
+  name = "dumpnet-fluentbit"
+}
+
+resource "aws_iam_access_key" "fluentbit" {
+  user = aws_iam_user.fluentbit.name
+}
+
+resource "aws_iam_user_policy" "fluentbit" {
+  name = "fluentbit-s3-write"
+  user = aws_iam_user.fluentbit.name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = ["s3:PutObject"]
+        Resource = "${aws_s3_bucket.logs.arn}/*"
+      }
+    ]
+  })
+}
+
+output "logs_bucket" {
+  value = aws_s3_bucket.logs.bucket
+}