update staging to have a second cluster and better procss isolation

k8s/applications/templates/ingress-nginx.yaml

@@ -12,6 +12,18 @@ spec:
     repoURL: 'https://kubernetes.github.io/ingress-nginx'
     targetRevision: 4.13.0
     chart: ingress-nginx
+    helm:
+      values: |
+        controller:
+          nodeSelector:
+            workload: infra
+          admissionWebhooks:
+            patch:
+              nodeSelector:
+                workload: infra
+        defaultBackend:
+          nodeSelector:
+            workload: infra
   project: default
   syncPolicy:
     syncOptions:

k8s/argocd/overlays/staging/kustomization.yaml

@@ -17,4 +17,20 @@ patchesJson6902:
       group: networking.k8s.io
       kind: Ingress
       name: argocd-server-ingress
-      version: v1
+      version: v1
+
+patches:
+- target:
+    kind: Deployment
+  patch: |-
+    - op: add
+      path: /spec/template/spec/nodeSelector
+      value:
+        workload: infra
+- target:
+    kind: StatefulSet
+  patch: |-
+    - op: add
+      path: /spec/template/spec/nodeSelector
+      value:
+        workload: infra

k8s/cert-manager/kustomization.yaml

@@ -4,4 +4,13 @@ kind: Kustomization
 
 resources:
   - https://github.com/jetstack/cert-manager/releases/download/v1.6.0/cert-manager.yaml
-  - cluster-issuer-nginx.yaml
+  - cluster-issuer-nginx.yaml
+
+patches:
+  - target:
+      kind: Deployment
+    patch: |-
+      - op: add
+        path: /spec/template/spec/nodeSelector
+        value:
+          workload: infra

k8s/coturn/templates/deployment.yml

@@ -43,3 +43,11 @@ spec:
             - name: coturn-config-volume
               mountPath: /etc/coturn/turnserver.conf
               subPath: turnserver.conf
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

k8s/coturn/values.yaml

@@ -12,6 +12,12 @@ imagePullSecrets: []
 
 redis:
   enabled: true
+  master:
+    nodeSelector:
+      workload: infra
+  replica:
+    nodeSelector:
+      workload: infra
   image:
     registry: public.ecr.aws
     repository: bitnami/redis
@@ -23,3 +29,11 @@ exporter:
     repository: ghcr.io/coturn/coturn-exporter
     tag: latest
     pullPolicy: IfNotPresent
+
+nodeSelector:
+  workload: media
+tolerations:
+- key: "dedicated"
+  operator: "Equal"
+  value: "media"
+  effect: "NoSchedule"

k8s/external-dns/overlays/staging/kustomization.yaml

@@ -12,4 +12,13 @@ patchesJson6902:
       group: apps
       version: v1
       kind: Deployment
-      name: external-dns
+      name: external-dns
+
+patches:
+  - target:
+      kind: Deployment
+    patch: |-
+      - op: add
+        path: /spec/template/spec/nodeSelector
+        value:
+          workload: infra

k8s/loki/values.yaml

@@ -64,6 +64,8 @@ loki:
 
   singleBinary:
     replicas: 1
+    nodeSelector:
+      workload: infra
     persistence:
       enabled: true
       size: 20Gi

k8s/monitoring/values-staging.yaml

@@ -5,6 +5,17 @@ rbac:
 
 
 kube-prometheus-stack:
+  prometheus:
+    prometheusSpec:
+      nodeSelector:
+        workload: infra
+  grafana:
+    nodeSelector:
+      workload: infra
+  alertmanager:
+    alertmanagerSpec:
+      nodeSelector:
+        workload: infra
   crds:
     enabled: false
 

k8s/promtail/values.yaml

@@ -2,3 +2,8 @@ promtail:
   config:
     clients:
       - url: http://loki.loki.svc:3100/loki/api/v1/push
+  tolerations:
+    - key: dedicated
+      operator: Equal
+      value: media
+      effect: NoSchedule

k8s/webrtc-be/templates/deployment.yml

@@ -68,6 +68,14 @@ spec:
             periodSeconds: 10
             timeoutSeconds: 5
             failureThreshold: 3
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
           readinessProbe:
             httpGet:
               path: /healthcheck

k8s/webrtc-be/values-staging.yaml

@@ -15,3 +15,11 @@ mediasoup:
 # Session Timeouts (ms)
 sessionHealthCheckInterval: 60000
 sessionOverGracePeriod: 300000
+
+nodeSelector:
+  workload: media
+tolerations:
+- key: "dedicated"
+  operator: "Equal"
+  value: "media"
+  effect: "NoSchedule"

terraform/lke.tf

@@ -10,7 +10,12 @@ resource "linode_lke_cluster" "stg-video-cluster" {
     type  = "g6-standard-2"
     count = 3
     labels = {
-        "workload" = "any"
+        "workload" = "media"
+    }
+    taint {
+      key    = "dedicated"
+      value  = "media"
+      effect = "NoSchedule"
     }
     autoscaler {
       min = 3
@@ -18,6 +23,18 @@ resource "linode_lke_cluster" "stg-video-cluster" {
     }
   }
 
+  pool {
+    type  = "g6-standard-1"
+    count = 2
+    labels = {
+        "workload" = "infra"
+    }
+    autoscaler {
+      min = 2
+      max = 5
+    }
+  }
+
 }
 
 resource "local_file" "kubeconfig" {