diff --git a/Makefile b/Makefile index e384bf8..05ba3c6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: init plan apply destroy kubeconfig talosconfig argocd-password bootstrap decrypt post-apply +.PHONY: init plan apply destroy kubeconfig talosconfig argocd-password bootstrap decrypt post-apply fluentbit-secret -SECRET=dumpnet/cluster +SECRET=dumpnet TF_DIR=terraform SOPS_FILES=controlplane.yaml worker.yaml talosconfig @@ -25,7 +25,7 @@ bootstrap: talosconfig: aws secretsmanager get-secret-value --secret-id $(SECRET) \ --query SecretString --output text | python3 -c \ - "import sys,json; print(json.load(sys.stdin)['talosconfig'])" \ + "import sys,json; print(json.load(sys.stdin)['cluster']['talosconfig'])" \ > /tmp/talosconfig-dumpnet talosctl config merge /tmp/talosconfig-dumpnet rm /tmp/talosconfig-dumpnet @@ -34,7 +34,7 @@ talosconfig: kubeconfig: aws secretsmanager get-secret-value --secret-id $(SECRET) \ --query SecretString --output text | python3 -c \ - "import sys,json; print(json.load(sys.stdin)['kubeconfig'])" \ + "import sys,json; print(json.load(sys.stdin)['cluster']['kubeconfig'])" \ > /tmp/kubeconfig-dumpnet KUBECONFIG=~/.kube/config:/tmp/kubeconfig-dumpnet kubectl config view --flatten > /tmp/merged mv /tmp/merged ~/.kube/config @@ -44,7 +44,7 @@ kubeconfig: argocd-password: @aws secretsmanager get-secret-value --secret-id $(SECRET) \ --query SecretString --output text | python3 -c \ - "import sys,json; print(json.load(sys.stdin)['argocd_admin_password'])" + "import sys,json; print(json.load(sys.stdin)['cluster']['argocd_admin_password'])" # Decrypt sensitive files to /tmp for one-off talosctl use # Files are never decrypted in the repo directory @@ -57,6 +57,20 @@ decrypt: done @echo "Decrypted files are in /tmp/ - they will not persist after reboot" +# Create fluent-bit AWS credentials secret in cluster +fluentbit-secret: + @KEY_ID=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \ + --query SecretString --output text | python3 -c \ + "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_access_key_id'])") && \ + SECRET_KEY=$$(aws secretsmanager get-secret-value --secret-id $(SECRET) \ + --query SecretString --output text | python3 -c \ + "import sys,json; print(json.load(sys.stdin)['fluentbit']['aws_secret_access_key'])") && \ + kubectl create secret generic fluentbit-aws-credentials \ + --namespace fluent-bit \ + --from-literal=AWS_ACCESS_KEY_ID=$$KEY_ID \ + --from-literal=AWS_SECRET_ACCESS_KEY=$$SECRET_KEY \ + --dry-run=client -o yaml | kubectl apply -f - + # Run all post-apply steps (run once after fresh cluster creation) post-apply: scripts/post-apply.sh diff --git a/apps/apps.yaml b/apps/apps.yaml index 4bda82e..ddb44f2 100644 --- a/apps/apps.yaml +++ b/apps/apps.yaml @@ -1,14 +1,14 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: apps + name: cluster namespace: argocd spec: project: default source: repoURL: https://forge.keane.sh/ian/dumpnet-argo.git targetRevision: HEAD - path: manifests + path: manifests/cluster destination: server: https://kubernetes.default.svc namespace: argocd @@ -16,3 +16,23 @@ spec: automated: prune: true selfHeal: true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: services + namespace: argocd +spec: + project: default + source: + repoURL: https://forge.keane.sh/ian/dumpnet-argo.git + targetRevision: HEAD + path: manifests/services + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml new file mode 100644 index 0000000..67a2916 --- /dev/null +++ b/charts/fluent-bit/values.yaml @@ -0,0 +1,56 @@ +## Fluent Bit Helm values +## AWS credentials are expected in a secret: fluent-bit/fluentbit-aws-credentials +## with keys: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +## Create with: make fluentbit-secret + +kind: DaemonSet + +env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: fluentbit-aws-credentials + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: fluentbit-aws-credentials + key: AWS_SECRET_ACCESS_KEY + +config: + service: | + [SERVICE] + Flush 5 + Daemon off + Log_Level warn + Parsers_File parsers.conf + + inputs: | + [INPUT] + Name tail + Tag kube.* + Path /var/log/containers/*.log + Parser cri + Mem_Buf_Limit 5MB + Skip_Long_Lines On + + filters: | + [FILTER] + Name kubernetes + Match kube.* + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + + outputs: | + [OUTPUT] + Name s3 + Match kube.* + bucket dumpnet-logs + region us-east-1 + s3_key_format /logs/%Y/%m/%d/$TAG[4].%H%M%S.gz + total_file_size 100M + upload_timeout 10m + compression gzip + use_put_object On diff --git a/manifests/argocd.yaml b/manifests/cluster/argocd.yaml similarity index 100% rename from manifests/argocd.yaml rename to manifests/cluster/argocd.yaml diff --git a/manifests/cert-manager.yaml b/manifests/cluster/cert-manager.yaml similarity index 100% rename from manifests/cert-manager.yaml rename to manifests/cluster/cert-manager.yaml diff --git a/manifests/cluster/fluent-bit.yaml b/manifests/cluster/fluent-bit.yaml new file mode 100644 index 0000000..e8128ee --- /dev/null +++ b/manifests/cluster/fluent-bit.yaml @@ -0,0 +1,26 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: fluent-bit + namespace: argocd +spec: + project: default + sources: + - repoURL: https://forge.keane.sh/ian/dumpnet-argo.git + targetRevision: HEAD + ref: values + - repoURL: https://fluent.github.io/helm-charts + chart: fluent-bit + targetRevision: "*" + helm: + valueFiles: + - $values/charts/fluent-bit/values.yaml + destination: + server: https://kubernetes.default.svc + namespace: fluent-bit + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/manifests/ingress-nginx.yaml b/manifests/cluster/ingress-nginx.yaml similarity index 100% rename from manifests/ingress-nginx.yaml rename to manifests/cluster/ingress-nginx.yaml diff --git a/manifests/metrics-server.yaml b/manifests/cluster/metrics-server.yaml similarity index 100% rename from manifests/metrics-server.yaml rename to manifests/cluster/metrics-server.yaml diff --git a/manifests/services/.gitkeep b/manifests/services/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/terraform/argocd.tf b/terraform/argocd.tf index e0ba870..47106a9 100644 --- a/terraform/argocd.tf +++ b/terraform/argocd.tf @@ -5,18 +5,24 @@ resource "random_password" "argocd_admin" { override_special = "!#$%&*()-_=+[]{}?" } -# Store secrets in AWS Secrets Manager +# Single secrets manager secret for all dumpnet secrets resource "aws_secretsmanager_secret" "dumpnet" { - name = "dumpnet/cluster" - description = "Dumpnet cluster secrets" + name = "dumpnet" + description = "All dumpnet secrets" } resource "aws_secretsmanager_secret_version" "dumpnet" { secret_id = aws_secretsmanager_secret.dumpnet.id secret_string = jsonencode({ - argocd_admin_password = random_password.argocd_admin.result - talosconfig = data.talos_client_configuration.this.talos_config - kubeconfig = talos_cluster_kubeconfig.this.kubeconfig_raw + cluster = { + argocd_admin_password = random_password.argocd_admin.result + talosconfig = data.talos_client_configuration.this.talos_config + kubeconfig = talos_cluster_kubeconfig.this.kubeconfig_raw + } + fluentbit = { + aws_access_key_id = aws_iam_access_key.fluentbit.id + aws_secret_access_key = aws_iam_access_key.fluentbit.secret + } }) } diff --git a/terraform/logging.tf b/terraform/logging.tf new file mode 100644 index 0000000..bdafea2 --- /dev/null +++ b/terraform/logging.tf @@ -0,0 +1,86 @@ +# S3 bucket for logs +resource "aws_s3_bucket" "logs" { + bucket = "dumpnet-logs" +} + +resource "aws_s3_bucket_lifecycle_configuration" "logs" { + bucket = aws_s3_bucket.logs.id + + rule { + id = "expire-old-logs" + status = "Enabled" + + filter {} + + transition { + days = 30 + storage_class = "INTELLIGENT_TIERING" + } + + expiration { + days = 365 + } + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "logs" { + bucket = aws_s3_bucket.logs.id + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +# Athena database + workgroup +resource "aws_athena_database" "logs" { + name = "dumpnet_logs" + bucket = aws_s3_bucket.athena_results.bucket +} + +resource "aws_s3_bucket" "athena_results" { + bucket = "dumpnet-athena-results" +} + +resource "aws_athena_workgroup" "logs" { + name = "dumpnet-logs" + + configuration { + result_configuration { + output_location = "s3://${aws_s3_bucket.athena_results.bucket}/results/" + + encryption_configuration { + encryption_option = "SSE_S3" + } + } + } +} + +# IAM user for fluent-bit to write to S3 +resource "aws_iam_user" "fluentbit" { + name = "dumpnet-fluentbit" +} + +resource "aws_iam_access_key" "fluentbit" { + user = aws_iam_user.fluentbit.name +} + +resource "aws_iam_user_policy" "fluentbit" { + name = "fluentbit-s3-write" + user = aws_iam_user.fluentbit.name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["s3:PutObject"] + Resource = "${aws_s3_bucket.logs.arn}/*" + } + ] + }) +} + +output "logs_bucket" { + value = aws_s3_bucket.logs.bucket +}