diff --git a/k8s/monitoring/values-staging.yaml b/k8s/monitoring/values-staging.yaml index eb663a7..c65047b 100644 --- a/k8s/monitoring/values-staging.yaml +++ b/k8s/monitoring/values-staging.yaml @@ -1,38 +1,43 @@ # Helm chart values for Prometheus Operator with HTTPS and basic auth +# Explicitly enable RBAC resource creation +rbac: + create: true + + kube-prometheus-stack: - nodeExporter: - enabled: true + + # Disable the default ServiceMonitor configuration paths to prevent duplicates + prometheus-node-exporter: serviceMonitor: - enabled: true - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - targetLabel: instance - action: replace - # Optional Rule 2: If the original IP address needs to be retained - # as a separate label (e.g., 'ip_address'), this rule can be added: - - sourceLabels: [__address__] - targetLabel: ip_address - action: replace + enabled: false + nodeExporter: + serviceMonitor: + enabled: false + prometheus: ingress: enabled: true + pathType: Prefix annotations: kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/rewrite-target: /$2 + #nginx.ingress.kubernetes.io/rewrite-target: /$2 cert-manager.io/cluster-issuer: letsencrypt-nginx-production - # nginx.ingress.kubernetes.io/auth-type: basic - # nginx.ingress.kubernetes.io/auth-secret: basic-auth - # nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + nginx.ingress.kubernetes.io/auth-type: basic + nginx.ingress.kubernetes.io/auth-secret: monitoring-basic-auth + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' hosts: - monitoring.staging.video.jamkazam.com paths: - - /prometheus(/|$)(.*) + - /prometheus tls: - secretName: monitoring hosts: - monitoring.staging.video.jamkazam.com prometheusSpec: - routePrefix: / + retention: 60d + retentionSize: 20GB + routePrefix: /prometheus externalUrl: https://monitoring.staging.video.jamkazam.com/prometheus storageSpec: volumeClaimTemplate: @@ -40,28 +45,83 @@ kube-prometheus-stack: storageClassName: linode-block-storage-retain resources: requests: - storage: 10Gi + storage: 30Gi + # 2. !!! CRUCIAL: Ensure the default ServiceMonitor is ignored !!! + # This prevents duplicate metrics by telling Prometheus to ignore the default SM. + serviceMonitorSelector: + matchExpressions: + # Exclude the default node-exporter ServiceMonitor + - key: app.kubernetes.io/name + operator: NotIn + values: + # Use the label identified above + - prometheus-node-exporter + serviceMonitorNamespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - monitoring # Its own namespace + - webrtc-be # Your app's namespace + # Add the manual scrape configuration + additionalScrapeConfigs: + - job_name: 'node-exporter' + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + # 1. Filter: Precisely target the node-exporter service in the monitoring namespace. + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name] + separator: '/' + # Assuming the service name is 'monitoring-prometheus-node-exporter' + regex: 'monitoring/monitoring-prometheus-node-exporter' + action: keep + + # 2. Filter: Ensure we are targeting the standard port (usually 9100) + - source_labels: [__address__] + regex: '.*:9100$' + action: keep + # 3. THE FIX: Set the instance label correctly + - source_labels: [__meta_kubernetes_endpoint_node_name] + target_label: instance + action: replace + - source_labels: [__address__] + target_label: ip_address + action: replace + + # 4. Replicate standard labels for dashboard compatibility + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + # Ensure standard labels are present for dashboard compatibility + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_endpoint_node_name] + target_label: node alertmanager: ingress: enabled: true + pathType: Prefix annotations: kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/rewrite-target: /$2 + #nginx.ingress.kubernetes.io/rewrite-target: /$2 cert-manager.io/cluster-issuer: letsencrypt-nginx-production - # nginx.ingress.kubernetes.io/auth-type: basic - # nginx.ingress.kubernetes.io/auth-secret: basic-auth - # nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + nginx.ingress.kubernetes.io/auth-type: basic + nginx.ingress.kubernetes.io/auth-secret: monitoring-basic-auth + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' hosts: - monitoring.staging.video.jamkazam.com paths: - - /alertmanager(/|$)(.*) + - /alertmanager tls: - secretName: monitoring hosts: - monitoring.staging.video.jamkazam.com alertmanagerSpec: - routePrefix: / + routePrefix: /alertmanager externalUrl: https://monitoring.staging.video.jamkazam.com/alertmanager storage: volumeClaimTemplate: @@ -69,25 +129,27 @@ kube-prometheus-stack: storageClassName: linode-block-storage-retain resources: requests: - storage: 10Gi + storage: 30Gi grafana: persistence: enabled: true storageClassName: linode-block-storage-retain - size: 10Gi + size: 30Gi ingress: enabled: true + pathType: Prefix annotations: kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/rewrite-target: /$2 + #nginx.ingress.kubernetes.io/rewrite-target: /$2 cert-manager.io/cluster-issuer: letsencrypt-nginx-production - # nginx.ingress.kubernetes.io/auth-type: basic - # nginx.ingress.kubernetes.io/auth-secret: basic-auth - # nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + #nginx.ingress.kubernetes.io/auth-type: basic + #nginx.ingress.kubernetes.io/auth-secret: monitoring-basic-auth + #nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' hosts: - monitoring.staging.video.jamkazam.com - path: /grafana(/|$)(.*) + path: /grafana tls: - secretName: monitoring hosts: @@ -97,6 +159,7 @@ kube-prometheus-stack: domain: monitoring.staging.video.jamkazam.com root_url: "%(protocol)s://%(domain)s/grafana/" enable_gzip: "true" + serve_from_sub_path: true # Disable control plane metrics kubeEtcd: @@ -107,4 +170,4 @@ kube-prometheus-stack: kubeScheduler: enabled: false - \ No newline at end of file +