diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 83f0ba5..c7693b4 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -65,25 +65,66 @@ pipelines: - docker push "gcr.io/${GCLOUD_PROJECT}/coturn-dns:${VERSION}" services: - docker + + + # - step: Deploy haproxy ingress controller + # % helm install haproxy-ingress haproxy-ingress/haproxy-ingress\ + # --create-namespace --namespace ingress-controller\ + # --version 0.13.1\ + # -f k8s/haproxy/haproxy-ingress-values.yaml + + + # - step: Deploy cert-manager + # helm install \ + # cert-manager jetstack/cert-manager \ + # --namespace cert-manager \ + # --create-namespace \ + # --version v1.5.0 \ + # --set installCRDs=true + # $ kubectl apply -f k8s/cert-manager/cluster-issuer-production.yaml + + + + # - step: Deploy GCR credentials + # kubectl create secret docker-registry gcr-json-key \ + # --docker-server=gcr.io \ + # --docker-username=_json_key \ + # --docker-password="$(cat k8s/gcp.json)" \ + # --docker-email=any@valid.email + # kubectl patch serviceaccount default \ + # -p '{"imagePullSecrets": [{"name": "gcr-json-key"}]}' + +# Deploy nginx ingress controller +# helm install nginx-ingress stable/nginx-ingress + +# Deploy monitoring clusterissuer +# kubectl apply -f k8s/monitoring/clusterissuer.yaml + +# Deploy monitoring certificate +# kubectl apply -f k8s/monitoring/certificate.yaml + +# Deploy monitoring helm +# helm install \ +# monitoring stable/prometheus-operator \ +# -f k8s/monitoring/helm-values.yaml \ +# --namespace monitoring \ +# --set grafana.adminPassword=jamkazamMonitoring + + - step: name: Deploy to K8s - deployment: production + deployment: staging script: - AUTOSCALER_IMAGE="gcr.io/$GCLOUD_PROJECT/autoscaler:prod-0.1.$BITBUCKET_BUILD_NUMBER" - COTURN_DNS_IMAGE="gcr.io/$GCLOUD_PROJECT/coturn-dns:prod-0.1.$BITBUCKET_BUILD_NUMBER" - sed -i "s|{{linode_autoscaler_image}}|$AUTOSCALER_IMAGE|g" k8s/linode-autoscaler/webrtc-be-autoscaler.yaml - sed -i "s|{{linode_autoscaler_image}}|$AUTOSCALER_IMAGE|g" k8s/linode-autoscaler/coturn-autoscaler.yaml - sed -i "s|{{coturn_dns_image}}|$COTURN_DNS_IMAGE|g" k8s/coturn-dns/coturn-dns.yaml - - pipe: atlassian/kubectl-run:1.1.2 - variables: - KUBE_CONFIG: $KUBE_CONFIG - KUBECTL_COMMAND: 'apply' - RESOURCE_PATH: 'k8s/linode-autoscaler/' - - pipe: atlassian/kubectl-run:1.1.2 - variables: - KUBE_CONFIG: $KUBE_CONFIG - KUBECTL_COMMAND: 'apply' - RESOURCE_PATH: 'k8s/external-dns/' + # - pipe: atlassian/kubectl-run:1.1.2 + # variables: + # KUBE_CONFIG: $KUBE_CONFIG + # KUBECTL_COMMAND: 'apply' + # RESOURCE_PATH: 'k8s/linode-autoscaler/' - pipe: atlassian/kubectl-run:1.1.2 variables: KUBE_CONFIG: $KUBE_CONFIG diff --git a/k8s/cert-manager/certificate-production.yaml b/k8s/cert-manager/certificate-production.yaml new file mode 100644 index 0000000..486e283 --- /dev/null +++ b/k8s/cert-manager/certificate-production.yaml @@ -0,0 +1,10 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: production +spec: + secretName: production-certificate + issuerRef: + name: letsencrypt-production + dnsNames: + - video.jamkazam.com \ No newline at end of file diff --git a/k8s/coturn-dns/production-coturn-dns.yaml b/k8s/coturn-dns/production-coturn-dns.yaml new file mode 100644 index 0000000..c1a5d62 --- /dev/null +++ b/k8s/coturn-dns/production-coturn-dns.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coturn-dns +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + selector: + matchLabels: + app: coturn-dns + template: + metadata: + labels: + app: coturn-dns + spec: + containers: + - name: coturn-dns + image: {{coturn_dns_image}} + env: + - name: AWS_ACCESS_KEY_ID + value: "AKIA2SXEHOQFBQRGCSST" + - name: AWS_SECRET_ACCESS_KEY + value: "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" + - name: PYTHONUNBUFFERED + value: "1" + - name: HOSTED_ZONE + value: "Z00156242SK162FEXDPVF" + - name: CLUSTER_ID + value: "29062" + - name: POOL_ID + value: "49934" + - name: LINODE_TOKEN + value: "a821bb97039cbd8b259e19ef9f7ea7a4e295a7399e00709fc27cad2b1f3742f4" + - name: COTURN_DOMAIN_NAME + value: "coturn.video.jamkazam.com" + resources: + requests: + memory: 32Mi + limits: + memory: 32Mi diff --git a/k8s/coturn-dns/register-nodes.py b/k8s/coturn-dns/register-nodes.py index 10253c6..25bdf92 100644 --- a/k8s/coturn-dns/register-nodes.py +++ b/k8s/coturn-dns/register-nodes.py @@ -4,11 +4,12 @@ import boto3 import time import os -HOSTED_ZONE=os.environ['HOSTED_ZONE'] #"Z00156242SK162FEXDPVF" -CLUSTER_ID=os.environ['CLUSTER_ID'] #"29062" -POOL_ID=os.environ['POOL_ID'] #"49934" +HOSTED_ZONE=os.environ['HOSTED_ZONE'] +CLUSTER_ID=os.environ['CLUSTER_ID'] +POOL_ID=os.environ['POOL_ID'] LINODE_TOKEN=os.environ['LINODE_TOKEN'] TOKEN={"Authorization": "Bearer "+LINODE_TOKEN} +COTURN_DOMAIN_NAME=os.environ['COTURN_DOMAIN_NAME'] while(True): r = requests.get("https://api.linode.com/v4/lke/clusters/"+CLUSTER_ID+"/pools/"+POOL_ID, headers=TOKEN) @@ -17,7 +18,6 @@ while(True): for node in r.json()['nodes']: ip = requests.get("https://api.linode.com/v4/linode/instances/"+str(node['instance_id'])+"/ips", headers=TOKEN) - #print(ip.json()) ips.append({'Value': ip.json()['ipv4']['public'][0]['address']}) print("Node IPs: "+str(ips)) @@ -30,7 +30,7 @@ while(True): { 'Action': 'UPSERT', 'ResourceRecordSet': { - 'Name': 'coturn.staging.video.jamkazam.com', + 'Name': COTURN_DOMAIN_NAME, 'Type': 'A', 'TTL': 300, 'ResourceRecords': ips diff --git a/k8s/coturn-dns/coturn-dns.yaml b/k8s/coturn-dns/staging-coturn-dns.yaml similarity index 100% rename from k8s/coturn-dns/coturn-dns.yaml rename to k8s/coturn-dns/staging-coturn-dns.yaml diff --git a/k8s/external-dns/deployment.yaml b/k8s/external-dns/deployment.yaml deleted file mode 100644 index 3eafe33..0000000 --- a/k8s/external-dns/deployment.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: external-dns ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: external-dns -rules: -- apiGroups: [""] - resources: ["services","endpoints","pods"] - verbs: ["get","watch","list"] -- apiGroups: ["extensions","networking.k8s.io"] - resources: ["ingresses"] - verbs: ["get","watch","list"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["list","watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: external-dns-viewer -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: external-dns -subjects: -- kind: ServiceAccount - name: external-dns - namespace: default ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: external-dns -spec: - strategy: - type: Recreate - selector: - matchLabels: - app: external-dns - template: - metadata: - labels: - app: external-dns - spec: - serviceAccountName: external-dns - containers: - - name: external-dns - image: k8s.gcr.io/external-dns/external-dns:v0.7.6 - env: - - name: AWS_ACCESS_KEY_ID - value: "AKIA2SXEHOQFBQRGCSST" - - name: AWS_SECRET_ACCESS_KEY - value: "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" - args: - - --source=ingress - - --source=service - - --domain-filter=video.jamkazam.com # will make ExternalDNS see only the hosted zones matching provided domain, omit to process all available hosted zones - - --provider=aws - - --policy=upsert-only # would prevent ExternalDNS from deleting any records, omit to enable full synchronization - - --aws-zone-type=public # only look at public hosted zones (valid values are public, private or no value for both) - - --registry=txt - - --txt-owner-id=my-hostedzone-identifier - securityContext: - fsGroup: 65534 \ No newline at end of file diff --git a/k8s/gcp.json b/k8s/gcp.json new file mode 100644 index 0000000..f502785 --- /dev/null +++ b/k8s/gcp.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "tough-craft-276813", + "private_key_id": "a8092b39b4eb391e8b1e8ace86d5c463e049e711", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCy18xh+H6vH4qJ\ns0x7syo8rK+dEgy/24dUTqPb54KfBmtXPdKuGCT/ZsoWAPqRhpmbYYe1Po9wNe6E\nXstVCvFq5ev2olJFzauy24UI6bWaXkQX/OHXLho/rn/EJPdcwBBQZ6mtrv+rgLWQ\nhiAHFMeaQSfwGrXeNnKWuT/PlJmDvliORjzm94r9fywzhArJq/lFNh0JWLTHfzVT\n6nhHIrOCQ+6IAszVerU6G7VfTAKoEaFS1OeLFwlUyhwc3SPm7ceLxBqz25APo3qA\nZFYyfLe43XbmKw1gta/QnpnPUtp3Wrm7sk9xy/maLx6xagVaUsGLNjWnZCjaPTkw\npe7FHU5XAgMBAAECggEADBP635ryo00UBByxy6Db92EKMydm6QYga5csBcvqzGaY\nlTm9orhKt1zvxPCn+3AFq7K4gYsKEN/zjckBHmswxrFkcDGiMMilEd01bNarxxMa\nsiwH7IpWh3p3cn20nvTxpRx7Hxm0dpaorGwCebfziv1ffx2urqUqs/cq0hANFhKF\n7bNYiTY6/9ZwWvcorpeu59UgJat2f12+aRUjj3Iu459UlRs6IhfXW2cWhMVHVylF\ng500i7sLrBLAlqGq8HnHkHUcB6sWnrWMBQ3wyqcEnORjVI0Oumaz1tphPEmxBy1n\n12arKrQ3N7Iij6mG/EX9Ha7J3tbFgb5Z9Xn3EObEAQKBgQDXBqm+HMEh35C7Jx7l\nhKdwRx87LhmBgDfGSxrNV0D/O8AFTPIuSDNeYi473AvUjsmnd6tQvtNFD6v8U16k\nRSwrwAr1eM4b8CIZ+nnMKt0ah96E8TyOBdp5Xfs18M4ZL9yddOpVrIVlDiQBIuHR\nZKvYvklxyxi5Ut6UtcNkKSl9VwKBgQDU7BBG//WeGC4N8e61pxfh+oBiNx6RoBt8\n++GPmksRwZYPnHqCtli5GX4UTQIrTAeAzbOzqe5t6G7yPqnJqKfPQnzZEXVu7d51\nFFIU7WAIUPs7AyNKDsWRDQ73q3M2EN3VqjyMX6DuUeTPfASjI8CCju0FtDtzqdm+\nSWDVLDcXAQKBgFRE1DkhY782sq3mAwHIHyateNvkkTJjYXhg7rwSufJNJE/ve+oP\nebI/oAbtkeVXoEf1ajpWzs19+tUEh06xnUH4HVNeaMgiL/smYp1VHxnKrbZEJIs0\nWA7AejcFjH/qdfdvXnb9Cbo09H9NgFpjrcVfrcDe622VwI1fPpf+Wbg5AoGBAIqo\nvKTwFU0CZCOStSi5CzWPw8GyMYcWZDBNfAPfsBl9HzNFbQbopvjL4C5qRApcNdqs\nmuVaubn7jxzUsA9ydO3lV5ao5vf5klBejmGwgESKMEGq9nVJD2I5xdCGZ74C1+RI\nO6wSrqPk0wRHuGFhbAHaAAMh70GQkAt6j8PjSnEBAoGBAK04V8fXPbCBxLoRfMbT\nBjeutWad36oTDuvLoIsMRM1vCF1oxpL+j4+7+hbupQ/UMcLvPN9RmwgJTjxOPN5b\nThwUn6UHfNWlb0pQrw764gMV+3EZgbEzx7pAi8QNEY5gLL0Qd/34eIm1exHuPJtM\n+MLbJDdyJ3PEZL9YOB1uKyC6\n-----END PRIVATE KEY-----\n", + "client_email": "ansible-sa@tough-craft-276813.iam.gserviceaccount.com", + "client_id": "104334872115406805719", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/ansible-sa%40tough-craft-276813.iam.gserviceaccount.com" +} diff --git a/k8s/monitoring/certificate.yaml b/k8s/monitoring/certificate.yaml index 33186e1..ebe3189 100644 --- a/k8s/monitoring/certificate.yaml +++ b/k8s/monitoring/certificate.yaml @@ -8,7 +8,7 @@ spec: duration: 2160h # 90d renewBefore: 360h # 15d issuerRef: - name: letsencrypt-production + name: letsencrypt-monitoring kind: ClusterIssuer dnsNames: - monitoring.video.jamkazam.com \ No newline at end of file diff --git a/k8s/monitoring/clusterissuer.yaml b/k8s/monitoring/clusterissuer.yaml new file mode 100644 index 0000000..46cc54b --- /dev/null +++ b/k8s/monitoring/clusterissuer.yaml @@ -0,0 +1,19 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-monitoring +spec: + acme: + # You must replace this email address with your own. + # Let's Encrypt will use this to contact you about expiring + # certificates, and issues related to your account. + email: victor.barba.martin@toptal.com + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + # Secret resource that will be used to store the account's private key. + name: issuer-account-key + # Add a single challenge solver, HTTP01 using nginx + solvers: + - http01: + ingress: + class: nginx diff --git a/k8s/monitoring/namespace.yaml b/k8s/monitoring/namespace.yaml new file mode 100644 index 0000000..503fa1d --- /dev/null +++ b/k8s/monitoring/namespace.yaml @@ -0,0 +1 @@ +#TODO \ No newline at end of file diff --git a/k8s/prd-video-cluster-kubeconfig.yaml b/k8s/prd-video-cluster-kubeconfig.yaml new file mode 100644 index 0000000..a68eecd --- /dev/null +++ b/k8s/prd-video-cluster-kubeconfig.yaml @@ -0,0 +1,25 @@ + +apiVersion: v1 +kind: Config +preferences: {} + +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EZ3hPREUxTURRd01Wb1hEVE14TURneE5qRTFNRFF3TVZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTDh0ClBtclBFM0ZGaUUrbm1OYUFvME00Zmg3UUpoSkdxYjNORTdYdHdtUkJ5S2RpWkQzUm5MYjdEa0RKQ2tiS3RaTnAKRWFKWFZmamk4MTl4N25vZzlLSlFGS1hNNFEwWWZsK0paNkY3WnhuOGZEeittYXhEUUVWWE1IZ3QzMVNPVUJydgpGRTVNb21hYnhtM25DdkRGM2IxMnlvNWFIVTYvdklhSHpSQWV2WkI1TFQ2RlY0blRxKzlVU1k4aVdIYjJDei92CmZ1aUd2TGE2N0tqVUdBL1dCeVREZitWSnV6cXUrSHdTZHFKU0lQYlNNTkJuaHFmNC9YV1Ayb3VJN1h5bEtnbG4KSHg2dGw2RVBYWXpVVWNaY215RUowYnhlNDdiZWFLcWoyc1l5cGNnTVNKSjM1UXNRLzU4MUdaRUpLTWFSUmpjMQpHTm1aWGozY0RPK2p0RFJkWjBFQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZKVHp2aC9GNkJHWkt0cDRJOTdRZWN4UGgxbUVNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDVmVBMEw4dFhTaWZEYWJqMlBnSEcvdkxQdXh6SXdJZXk3NVptVU12NlBLaGxNcE4zSApTcXUrUnhrVlM3S3pJejE1Z2RzWG4rZm9lZ2hvWUdtaVZWVmx0aTNZejVNSU5tTXZEZksxWTFjQ2xzWXNCdzBnCjE0SHpGRUoxaEExdXh3UGFKdzdVUDJyWE5LYWtKMHZVOVVpTnpxM0laWkFZT2p1TGVQLzQyeHhBVUNQdGhSZ28KTW9qTlhrdjVvVTlKS3FycDIvSmd1ZFJhVExoSEJGQUsxVDJrc3U2TVA5NU5qUTFyOWhBK0tieEVnbk93d2tSLwppV2hvUEpNNTdSWFF1ZzJOVzZaU29TRUFhdWN4Qy90UEo5bldwWk9TYkRReFNJYXJjdWdyMzlsWWZ1L1l5ak5kCnExUWtiODYzekxhZitxVmJEY0ladGRZQ3djTnN5RjRLcUF6awotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + server: https://2c96c0af-61c1-4a4a-a7f7-3f5f4aa008fa.cpc2-us-central.linodelke.net:443 + name: lke35025 + +users: +- name: lke35025-admin + user: + as-user-extra: {} + token: eyJhbGciOiJSUzI1NiIsImtpZCI6IkJkektZakZheTZHYVhMcjJ6YWlpdGpUQ1dxMWpEdlhCYi1JN05KNnBsVDAifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJsa2UtYWRtaW4tdG9rZW4tNTVuOWwiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoibGtlLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiYWMxNjhkODQtZmNkOC00MTFiLTk5M2MtYzQyMjc4NTEwODU1Iiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmxrZS1hZG1pbiJ9.C-Co-E_coNyRqtRImZoR7IpPgNyZUYcF5ERMkjRWNS1r5mb7aETeduZkUQ6nEEgbke50tjt3MgHV0VFZr8rx22JiXMI6K6KmhPRR4aCznf9jFOi0Uja7sRLAeDDuVqn7Z0a858gTswkGPlBZCFyBzKUj6NGGpOpTzYb1Y8-AZF6ns8DaVntp6n1yFQXtQ3hXfAzfaF_JRREjn1mTJsV9025rlyGrGJ5tNGgnlQsIOYTs3GxFvE-LImw2041MLc875F21N78YzzaMC7ujh7PsXw2UVfHZknHRUwojd9see7C0kGf0W7xPSPtxXyEC1Cyp7YPGHSbUPulI0oiKNATCMA + +contexts: +- context: + cluster: lke35025 + namespace: default + user: lke35025-admin + name: lke35025-ctx + +current-context: lke35025-ctx diff --git a/k8s/video-cluster-kubeconfig.yaml b/k8s/stg-video-cluster-kubeconfig.yaml similarity index 100% rename from k8s/video-cluster-kubeconfig.yaml rename to k8s/stg-video-cluster-kubeconfig.yaml diff --git a/terraform/lke.tf b/terraform/lke.tf index be79b52..bf6e232 100644 --- a/terraform/lke.tf +++ b/terraform/lke.tf @@ -22,3 +22,27 @@ resource "linode_lke_cluster" "my-cluster" { } +resource "linode_lke_cluster" "prd-video-cluster" { + label = "prd-video-cluster" + k8s_version = "1.21" + region = "us-central" + tags = ["production"] + + pool { + type = "g6-standard-2" + count = 3 + } + + # WebRTC-BE pool + pool { + type = "g6-standard-2" + count = 3 + } + # Coturn pool + pool { + type = "g6-standard-2" + count = 3 + } + +} +