Enable carbonapi k8 deployment (#157)

gtema · web-flow · commit 3d5d249fc087 · 2021-06-07T11:11:15.000Z
Enable carbonapi k8 deployment

Reviewed-by: None &lt;None&gt;
Reviewed-by: kucerakk &lt;kucerakk@gmail.com&gt;
diff --git a/inventory/service/group_vars/carbonapi.yaml b/inventory/service/group_vars/carbonapi.yaml
@@ -1,15 +1,22 @@
 # TODO: This should be better constructed from existing inventory
-carbonapi_backends:
-  - groupName: "apimon"
-    lbMethod: "any"
-    protocol: "carbonapi_v3_pb"
-    servers:
-      - "http://192.168.191.55:8081"
-      - "http://192.168.14.159:8081"
-      - "http://192.168.151.11:8081"
-  - groupName: "zuul"
-    lbMethod: "broadcast"
-    protocol: "carbonapi_v3_pb"
-    servers:
-      - "http://192.168.14.241:8081"
-
+carbonapi_instances:
+  main:
+    fqdn: "graphite-ca.eco.tsi-dev.otc-service.com"
+    carbonapi_cert: "graphite"
+    backends:
+      - groupName: "apimon"
+        lbMethod: "any"
+        protocol: "carbonapi_v3_pb"
+        servers:
+          - "http://192.168.191.55:8081"
+          - "http://192.168.14.159:8081"
+          - "http://192.168.151.11:8081"
+      - groupName: "zuul"
+        lbMethod: "broadcast"
+        protocol: "carbonapi_v3_pb"
+        servers:
+          - "http://192.168.14.241:8081"
+    memcache_hosts:
+      - "192.168.191.55:11211"
+      - "192.168.14.159:11211"
+      - "192.168.151.11:11211"
diff --git a/inventory/service/group_vars/k8s-controller.yaml b/inventory/service/group_vars/k8s-controller.yaml
@@ -20,3 +20,14 @@ graphite_web_k8s_instances:
     context: "otcinfra-mirror"
     instance: "apimon"
     namespace: "graphite"
+
+carbonapi_k8s_instances:
+  - carbonapi_instance: "main"
+    context: "otcinfra"
+    instance: "main"
+    namespace: "graphite"
+  - carbonapi_instance: "main"
+    context: "otcinfra-mirror"
+    instance: "main"
+    namespace: "graphite"
+
diff --git a/inventory/service/group_vars/proxy.yaml b/inventory/service/group_vars/proxy.yaml
@@ -19,16 +19,15 @@ proxy_backends:
         address: "192.168.151.11:443"
         opts: "ssl verify none cookie graphite3 check sni req.hdr(Host)"
 
-  - name: "graphite-apimon2"
+  - name: "graphite-ca"
     options:
-      # NOTE: try to perform simliest query
-      - "httpchk HEAD /metrics/find/?query=*"
+      - "httpchk HEAD /metrics/lb_check"
     domain_names:
-      - "graphite.apimon.eco.tsi-dev.otc-service.com"
+      - "graphite-ca.eco.tsi-dev.otc-service.com"
     servers:
-      - name: "graphite-proxy1"
-        address: "192.168.110.160:8082"
-        opts: "verify none cookie graphite1 check sni req.hdr(Host)"
+      - name: "graphite-otcinfra"
+        address: "192.168.170.249:443"
+        opts: "ssl verify none cookie graphite-otcinfra check sni req.hdr(Host)"
       - name: "graphite-proxy2"
         address: "192.168.110.151:8082"
         opts: "verify none cookie graphite2 check sni req.hdr(Host)"
diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml
@@ -85,7 +85,7 @@ groups:
     - graphite1.eco.tsi-dev.otc-service.com
 
   carbonapi:
-    - proxy1.eco.tsi-dev.otc-service.com
+    - bridge.eco.tsi-dev.otc-service.com
     - proxy2.eco.tsi-dev.otc-service.com
     - web3.eco.tsi-dev.otc-service.com
 
diff --git a/inventory/service/host_vars/bridge.eco.tsi-dev.otc-service.com.yaml b/inventory/service/host_vars/bridge.eco.tsi-dev.otc-service.com.yaml
@@ -20,3 +20,5 @@ ssl_certs:
     - "swift.eco.tsi-dev.otc-service.com"
   graphite-apimon:
     - "graphite.apimon.eco.tsi-dev.otc-service.com"
+  graphite:
+    - "graphite-ca.eco.tsi-dev.otc-service.com"
diff --git a/inventory/service/host_vars/proxy1.eco.tsi-dev.otc-service.com.yaml b/inventory/service/host_vars/proxy1.eco.tsi-dev.otc-service.com.yaml
@@ -6,6 +6,9 @@ ssl_certs:
     - "apimon.eco.tsi-dev.otc-service.com"
     - "alerts.eco.tsi-dev.otc-service.com"
     - "graphite.eco.tsi-dev.otc-service.com"
+    - "graphite-ca.eco.tsi-dev.otc-service.com"
     - "influx1.eco.tsi-dev.otc-service.com"
 
 firewalld_extra_services_enable: ['http', 'https']
+
+carbonapi_instance: "main"
diff --git a/inventory/service/host_vars/proxy2.eco.tsi-dev.otc-service.com.yaml b/inventory/service/host_vars/proxy2.eco.tsi-dev.otc-service.com.yaml
@@ -6,6 +6,9 @@ ssl_certs:
     - "apimon.eco.tsi-dev.otc-service.com"
     - "alerts.eco.tsi-dev.otc-service.com"
     - "graphite.eco.tsi-dev.otc-service.com"
+    - "graphite-ca.eco.tsi-dev.otc-service.com"
     - "influx1.eco.tsi-dev.otc-service.com"
 
 firewalld_extra_services_enable: ['http', 'https']
+
+carbonapi_instance: "main"
diff --git a/inventory/service/host_vars/web3.eco.tsi-dev.otc-service.com.yaml b/inventory/service/host_vars/web3.eco.tsi-dev.otc-service.com.yaml
@@ -12,10 +12,13 @@ ssl_certs:
     - "alerts.eco.tsi-dev.otc-service.com"
   graphite:
     - "graphite.eco.tsi-dev.otc-service.com"
+    - "graphite-ca.eco.tsi-dev.otc-service.com"
     - "graphite.apimon.eco.tsi-dev.otc-service.com"
 
 graphite_instance_group: "graphite-apimon"
 graphite_cert: "graphite"
 
 firewalld_extra_services_enable: ['http', 'https']
 firewalld_extra_ports_enable: ['3000/tcp', '8080/tcp', '8081/tcp']
+
+carbonapi_instance: "main"
diff --git a/playbooks/roles/acme_request_certs/tasks/acme.yaml b/playbooks/roles/acme_request_certs/tasks/acme.yaml
@@ -10,6 +10,7 @@
     cert: "{{ certs_path }}/{{ cert.key }}.crt"
     challenge: "dns-01"
     remaining_days: 60
+    force: "{{ csr_result is changed }}"
   register: challenge
 
 - name: Save acme challenge
diff --git a/playbooks/roles/acme_request_certs/tasks/common.yaml b/playbooks/roles/acme_request_certs/tasks/common.yaml
@@ -9,3 +9,4 @@
     privatekey_path: "{{ certs_path }}/{{ cert.key }}.pem"
     common_name: "{{ cert.value[0] }}"
     subject_alt_name: "DNS:{{ cert.value | join(',DNS:') }}"
+  register: csr_result
diff --git a/playbooks/roles/carbonapi/defaults/main.yaml b/playbooks/roles/carbonapi/defaults/main.yaml
@@ -1,7 +1,26 @@
 graphite_config_location: "/etc/graphite"
-graphite_tz: "Europe/Berlin"
+carbonapi_tz: ""
+  # "Europe/Berlin"
 
 carbonapi_image: "gographite/carbonapi:v0.15.3"
 
 container_command: "podman"
 container_runtime: "/usr/bin/{{ container_command }}"
+
+state: present
+carbonapi_count: 1
+carbonapi_cm_name: "carbonapi-{{ instance }}"
+carbonapi_svc_name: "carbonapi-{{ instance }}"
+carbonapi_dpl_name: "carbonapi-{{ instance }}"
+
+carbonapi_resources:
+  limits:
+    cpu: 1
+    memory: 2G
+  requests:
+    cpu: 100m
+    memory: 500Mi
+carbonapi_readiness_timeout: 10
+carbonapi_liveness_timeout: 5
+
+
diff --git a/playbooks/roles/carbonapi/tasks/k8s.yaml b/playbooks/roles/carbonapi/tasks/k8s.yaml
@@ -0,0 +1,183 @@
+---
+- name: Create carbonapi Namespace
+  community.kubernetes.k8s:
+    context: "{{ context }}"
+    state: "present"
+    name: "{{ namespace }}"
+    api_version: "v1"
+    kind: "Namespace"
+
+- name: Create carbonapi ConfigMap
+  community.kubernetes.k8s:
+    context: "{{ context }}"
+    state: "{{ state }}"
+    namespace: "{{ namespace }}"
+    name: "{{ carbonapi_cm_name }}"
+    kind: "ConfigMap"
+    api_version: "v1"
+    definition:
+      metadata:
+        labels:
+          app.kubernetes.io/name: "carbonapi"
+          app.kubernetes.io/instance: "{{ instance }}"
+          app.kubernetes.io/managed-by: "system-config"
+      data:
+        carbonapi.yaml: "{{ lookup('template', 'carbonapi.yaml.j2') }}"
+
+- name: Create carbonapi TLS Certs Secret
+  community.kubernetes.k8s:
+    context: "{{ context }}"
+    state: "present"
+    namespace: "{{ namespace }}"
+    name: "carbonapi-{{ instance }}-tls"
+    api_version: "v1"
+    kind: "Secret"
+    definition:
+      metadata:
+        labels:
+          app.kubernetes.io/app: "carbonapi"
+          app.kubernetes.io/instance: "{{ instance }}"
+          app.kubernetes.io/managed-by: "system-config"
+      data:
+        tls.crt: "{{ lookup('file', '/etc/ssl/{{ inventory_hostname }}/{{ carbonapi.carbonapi_cert }}-fullchain.crt') | string | b64encode }}"
+        tls.key: "{{ lookup('file', '/etc/ssl/{{ inventory_hostname }}/{{ carbonapi.carbonapi_cert }}.pem') | string | b64encode }}"
+
+- name: Create carbonapi Service
+  community.kubernetes.k8s:
+    context: "{{ context }}"
+    state: "{{ state }}"
+    namespace: "{{ namespace }}"
+    name: "{{ carbonapi_svc_name }}"
+    kind: "Service"
+    api_version: "v1"
+    apply: "yes"
+    definition:
+      metadata:
+        labels:
+          app.kubernetes.io/name: "carbonapi"
+          app.kubernetes.io/instance: "{{ instance }}"
+          app.kubernetes.io/managed-by: "system-config"
+      spec:
+        type: "ClusterIP"
+        ports:
+          - name: "carbonapi"
+            port: 8081
+            protocol: "TCP"
+            targetPort: 8081
+        selector:
+          app.kubernetes.io/name: "carbonapi"
+          app.kubernetes.io/instance: "{{ instance }}"
+
+- name: Create carbonapi Deployment
+  community.kubernetes.k8s:
+    context: "{{ context }}"
+    state: "{{ state }}"
+    namespace: "{{ namespace }}"
+    name: "{{ carbonapi_dpl_name }}"
+    api_version: "apps/v1"
+    kind: "Deployment"
+    apply: "yes"
+    definition:
+      metadata:
+        labels:
+          app.kubernetes.io/name: "carbonapi"
+          app.kubernetes.io/instance: "{{ instance }}"
+          app.kubernetes.io/managed-by: "system-config"
+      spec:
+        selector:
+          matchLabels:
+            app.kubernetes.io/name: "carbonapi"
+            app.kubernetes.io/instance: "{{ instance }}"
+        serviceName: "{{ carbonapi_svc_name }}"
+        template:
+          metadata:
+            labels:
+              app.kubernetes.io/name: "carbonapi"
+              app.kubernetes.io/instance: "{{ instance }}"
+          spec:
+            containers:
+              - resources: "{{ carbonapi_resources | default(omit) }}"
+                readinessProbe:
+                  initialDelaySeconds: 10
+                  httpGet:
+                    path: "/metrics/find/?query=*"
+                    port: "carbonapi"
+                  timeoutSeconds: "{{ carbonapi_readiness_timeout }}"
+                livenessProbe:
+                  initialDelaySeconds: 10
+                  httpGet:
+                    path: "/metrics/find/?query=*"
+                    port: "carbonapi"
+                  timeoutSeconds: "{{ carbonapi_liveness_timeout }}"
+                name: "carbonapi"
+                ports:
+                  - name: "carbonapi"
+                    containerPort: 8081
+                    protocol: "TCP"
+                volumeMounts:
+                  - name: "carbonapi-configmap"
+                    mountPath: "/etc/carbonapi.yml"
+                    subPath: "carbonapi.yaml"
+                image: "{{ carbonapi_image }}"
+            volumes:
+              - name: "carbonapi-configmap"
+                configMap:
+                  name: "{{ carbonapi_cm_name }}"
+                  defaultMode: 420
+
+- name: Create carbonapi Autoscaler
+  community.kubernetes.k8s:
+    context: "{{ context }}"
+    state: "present"
+    namespace: "{{ namespace }}"
+    name: "carbonapi-{{ instance }}"
+    api_version: "autoscaling/v2beta2"
+    kind: "HorizontalPodAutoscaler"
+    apply: "yes"
+    definition:
+      spec:
+        scaleTargetRef:
+          kind: "Deployment"
+          name: "{{ carbonapi_dpl_name }}"
+          apiVersion: "apps/v1"
+        minReplicas: 1
+        maxReplicas: 4
+        metrics:
+          - type: "Resource"
+            resource:
+              name: "cpu"
+              target:
+                type: "Utilization"
+                averageUtilization: 70
+
+- name: Create carbonapi Ingress
+  community.kubernetes.k8s:
+    context: "{{ context }}"
+    state: "present"
+    namespace: "{{ namespace }}"
+    name: "carbonapi-{{ instance }}"
+    api_version: "networking.k8s.io/v1"
+    kind: "Ingress"
+    apply: "yes"
+    definition:
+      metadata:
+        labels:
+          app.kubernetes.io/name: "carbonapi"
+          app.kubernetes.io/instance: "{{ instance }}"
+          app.kubernetes.io/managed-by: "system-config"
+      spec:
+        tls:
+          - hosts:
+              - "{{ carbonapi.fqdn }}"
+            secretName: "carbonapi-{{ instance }}-tls"
+        rules:
+          - host: "{{ carbonapi.fqdn }}"
+            http:
+              paths:
+                - path: "/"
+                  pathType: "Prefix"
+                  backend:
+                    service:
+                      name: "carbonapi-{{ instance }}"
+                      port:
+                        number: 8081
diff --git a/playbooks/roles/carbonapi/tasks/main.yaml b/playbooks/roles/carbonapi/tasks/main.yaml
@@ -29,6 +29,9 @@
     - "{{ graphite_config_location }}"
     - "/var/log/graphite"
 
+- debug:
+    var: carbonapi
+
 - name: Write config files
   become: true
   ansible.builtin.template:
diff --git a/playbooks/roles/carbonapi/templates/carbonapi.yaml.j2 b/playbooks/roles/carbonapi/templates/carbonapi.yaml.j2
@@ -39,17 +39,17 @@ notFoundStatusCode: 200
 concurency: 1000
 cache:
    # Type of caching. Valid: "mem", "memcache", "null"
-   type: {{ "memcache" if memcache_hosts else "mem" }}
+   type: {{ "memcache" if carbonapi.memcache_hosts else "mem" }}
    # Cache limit in megabytes
    size_mb: 32
    # Default cache timeout value. Identical to DEFAULT_CACHE_DURATION in graphite-web.
    defaultTimeoutSec: 60
    # Only used by memcache type of cache. List of memcache servers.
-   memcachedServers: {{ memcache_hosts | list }}
+   memcachedServers: {{ carbonapi.memcache_hosts | list }}
 # Amount of CPUs to use. 0 - unlimited
 cpus: 0
 # Timezone, default - local
-tz: ""
+tz: "{{ carbonapi_tz }}"
 
 #functionsConfig:
 #    graphiteWeb: ./graphiteWeb.example.yaml
@@ -101,7 +101,7 @@ upstreams:
     # It MUST be specified.
     backends: 
     backendsv2:
-      backends: {{ carbonapi_backends }}
+      backends: {{ carbonapi.backends }}
 
     # Enable compatibility with graphite-web 0.9
     # This will affect graphite-web 1.0+ with multiple cluster_servers
diff --git a/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml b/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml
@@ -14,3 +14,4 @@ results:
     - zuul
     - nodepool
     - ssl_certs
+    - carbonapi
diff --git a/playbooks/service-graphite.yaml b/playbooks/service-graphite.yaml
diff --git a/playbooks/zuul/templates/group_vars/k8s-controller.yaml.j2 b/playbooks/zuul/templates/group_vars/k8s-controller.yaml.j2
