Merge pull request #1 from peterparser/kubernetes-monitoring

Kubernetes monitoring

Author: peterparser

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "args": ["-u", "https://prometheus-k8s-openshift-monitoring.apps.elclown.lab.local", "-n", "sso"]
+        }
+    ]
+}

README.md

@@ -3,9 +3,6 @@
 This tool simplifies the gathering of data about the resource consumption of your pods.
 In particular, it outputs an excel table with the current configuration of your deployments and statefulsets and their resources, also the tool estimates the requests and the limits based on prometheus data.
 
-# DISCLAIMER
-At the moment the tool has been tested only against Openshift Clusters, a more agnostic version is coming soon!
-
 # Requirements
 To run the script you need:
 * An authenticated kubectl session

main.py

@@ -36,14 +36,16 @@
         epilog='RRS')
 
     parser.add_argument("-u", "--url", help="Prometheus URL", type=str, required=True)
+    parser.add_argument("-t", "--type", choices=['kubernetes', 'openshift'], default="kubernetes", help="Kubernetes flavour", type=str)
     parser.add_argument("-n", "--namespaces", help="Comma separated list of namespaces", type=str, required=True)
     parser.add_argument("-o", "--output", default="output.xlsx", help="Output file", type=str)
+
     args = parser.parse_args()
     token = getpass.getpass("Insert SA Token here:")
 
     config.load_kube_config()
     retriever = Retriever()
-    suggester = Suggester(args.url, token)
+    suggester = Suggester(args.url, token, args.type)
     namespaces = args.namespaces.split(",")
 
     data = []

querier.py

@@ -0,0 +1,80 @@
+import requests
+
+from urllib3.exceptions import InsecureRequestWarning 
+
+from statistics import mean
+
+queries = {
+    "kubernetes": {
+        "memory_request": {
+            "query": "avg_over_time(container_memory_working_set_bytes{{pod=~'{workload}.*',namespace='{namespace}',container=''}}[{time_range}])",
+            "query_type": "query"
+        },
+        "memory_limit": {
+            "query": "max_over_time(container_memory_working_set_bytes{{pod=~'{workload}.*',namespace='{namespace}',container=''}}[{time_range}])",
+            "query_type": "query"
+        },
+        "cpu_request": {
+            "query": "avg(rate(container_cpu_usage_seconds_total{{pod=~'{workload}.*',namespace='{namespace}', container=''}}[1m]))",
+            "query_type": "query_range"
+        },
+        "cpu_limit": {
+            "query": "avg(rate(container_cpu_usage_seconds_total{{pod=~'{workload}.*',namespace='{namespace}', container=''}}[1m]))",
+            "query_type": "query_range"
+        }
+
+    },
+    "openshift": {
+        "memory_request": {
+            "query": "avg_over_time(container_memory_working_set_bytes{{pod=~'{workload}.*',namespace='{namespace}',container=''}}[{time_range}])",
+            "query_type": "query"
+        },
+        "memory_limit": {
+            "query": "max_over_time(container_memory_working_set_bytes{{pod=~'{workload}.*',namespace='{namespace}',container=''}}[{time_range}])",
+            "query_type": "query"
+        },
+        "cpu_request": {
+            "query": "avg_over_time(pod:container_cpu_usage:sum{{pod=~'{workload}.*',namespace='{namespace}'}}[{time_range}])",
+            "query_type": "query"
+        },
+        "cpu_limit": {
+            "query": "max_over_time(pod:container_cpu_usage:sum{{pod=~'{workload}.*',namespace='{namespace}'}}[{time_range}])",
+            "query_type": "query"
+        }
+    }
+
+}
+
+
+aggregators = {
+    "cpu_limit": max,
+    "cpu_request": mean
+}
+
+def handle_memory(result):
+    return f'{round(float(result[0]["value"][1]) / 1000000)} M'
+
+def handle_cpu(result, aggregator):
+    if result[0]["values"]:
+        values = map(lambda x: float(x[1]), result[0]["values"])
+
+        return  f'{round(aggregator(values), 2)}'
+    else:
+        return f'{round(float(result[0]["value"][1]), 2)}'
+    
+
+def handle_response(metric, result, aggregator=None):
+    
+    match metric:
+        case "memory_request" | "memory_limit":
+            return handle_memory(result)
+    
+    
+        case "cpu_request" | "cpu_limit":
+            return handle_cpu(result, aggregator)
+    
+
+def run_query(url, headers, params):
+    requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+    response = requests.get(url, headers=headers, params=params, verify=False)
+    return response.json()["data"]["result"]

retriever.py

@@ -10,7 +10,6 @@ def __init__(self):
         self.retrieve_function = {
             "deployment": self.resources.list_namespaced_deployment,
             "statefulset": self.resources.list_namespaced_stateful_set,
-            "daemonset": self.resources.list_namespaced_daemon_set
         }
 
     def get_mem_cpu_req_lim(self, namespace):
@@ -34,9 +33,19 @@ def get_mem_cpu_req_lim(self, namespace):
                     else:
                         limit_cpu = "Not Defined"
                         limit_memory = "Not Defined"
+
+                    if object_type == 'deployment':
+                        for status in k8s_object.status.conditions:
+                            if status.type == 'Progressing':
+                                replicaset_name = status.message.split(" ")[1].replace('"', "")
+                                regex = f'{replicaset_name}.*'
+                                break
+                    else:
+                        regex = f'{name}-[0-9]+'
+                    print(regex)
                     results.append((namespace, object_type, name,
                                     container.name, replicas,
                                     request_cpu, limit_cpu,
-                                    request_memory, limit_memory))
+                                    request_memory, limit_memory, regex))
 
         return results

suggester.py

@@ -1,54 +1,63 @@
-import requests
+import time
+
 from urllib3.exceptions import InsecureRequestWarning
 
+import querier
 
 class Suggester:
-
-    def __init__(self, prometheus_host, token, time_range='7d'):
+    def __init__(self, prometheus_host, token, k8s_flavour, time_range='7d', step='3h'):
         self.prometheus_host = prometheus_host
         self.token = token
         self.time_range = time_range
+        self.queries = querier.queries[k8s_flavour]
+        self.end = int(time.time())
+        self.start = self.end - 24 * 60 * 60 * int(time_range[:-1])
+        self.step = step
+        self.aggregator = {}
+
+        for metric, query_struct in self.queries.items():
+            if query_struct["query_type"] == 'query_range':
+                self.aggregator[metric] = querier.aggregators[metric]
+
+
+    def build_query(self, k8s_object, metric):
+        host = f"{self.prometheus_host}/api/v1/{self.queries[metric]['query_type']}"
+        match self.queries[metric]["query_type"]:
+            case "query":
+                params = {
+                    "query": self.queries[metric]["query"].format(
+                        workload=k8s_object[9],
+                        namespace=k8s_object[0],
+                        time_range=self.time_range
+                        )
+                }
+            case "query_range":
+                params = {
+                    "query": self.queries[metric]["query"].format(
+                        workload=k8s_object[9],
+                        namespace=k8s_object[0],
+                        time_range=self.time_range
+                        ),
+                    "start": self.start,
+                    "end": self.end,
+                    "step": self.step
+                }
+
+        return host, params
+
 
     def suggest_values(self, k8s_object):
-        requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+        
         headers = {
             'Authorization': f'Bearer {self.token}'
         }
-        #Pe i pod avg_over_time(container_memory_working_set_bytes{pod=~'snitch-jvm.*',namespace='snitch',container='snitch-jvm'}[7d])
-        # Retrieve pod memory usage
-        params_memory_request = {
-            "query": f"avg_over_time(container_memory_working_set_bytes{{pod=~'{k8s_object[2]}.*',namespace='{k8s_object[0]}',container=''}}[{self.time_range}])"
-        }
-
-        response = requests.get(f"{self.prometheus_host}/api/v1/query", headers=headers, params=params_memory_request, verify=False)
-        memoryRequest = f'{round(float(response.json()["data"]["result"][0]["value"][1]) / 1000000)} M'
-
-        params_memory_limit = {
-            "query": f"max_over_time(container_memory_working_set_bytes{{pod=~'{k8s_object[2]}.*',namespace='{k8s_object[0]}',container=''}}[{self.time_range}])"
-        }
-
-        response = requests.get(f"{self.prometheus_host}/api/v1/query", headers=headers, params=params_memory_limit,
-                                verify=False)
-
-        memoryLimit = f'{round(float(response.json()["data"]["result"][0]["value"][1]) / 1000000)} M'
-
-        params_cpu_request = {
-            "query": f"avg_over_time(pod:container_cpu_usage:sum{{pod=~'{k8s_object[2]}.*',namespace='{k8s_object[0]}'}}[{self.time_range}])"
-        }
-
-        response = requests.get(f"{self.prometheus_host}/api/v1/query", headers=headers, params=params_cpu_request,
-                                verify=False)
-
-        cpu_request = f'{round(float(response.json()["data"]["result"][0]["value"][1]), 3)}'
-
-        params_cpu_limit = {
-            "query": f"max_over_time(pod:container_cpu_usage:sum{{pod=~'{k8s_object[2]}.*',namespace='{k8s_object[0]}'}}[{self.time_range}])"
-        }
 
-        response = requests.get(f"{self.prometheus_host}/api/v1/query", headers=headers, params=params_cpu_limit,
-                                verify=False)
+        results = {}
 
-        cpu_limit = f'{round(float(response.json()["data"]["result"][0]["value"][1]), 3)}'
+        for metric, query_struct in self.queries.items():
+            host, params = self.build_query(k8s_object, metric)
+            result = querier.run_query(host, headers, params)
+            results[metric] = querier.handle_response(metric, result, self.aggregator.get(metric, None))
 
         return [
             k8s_object[0], # Namespace
@@ -57,13 +66,13 @@ def suggest_values(self, k8s_object):
             k8s_object[3], # Container
             k8s_object[4], # Replicas
             k8s_object[5], # Request CPU
-            cpu_request, # suggested
+            results["cpu_request"], # suggested
             k8s_object[6], # CPU LIMIT,
-            cpu_limit,
+            results["cpu_limit"],
             k8s_object[7], # Memory request
-            memoryRequest, # suggest
+            results["memory_request"], # suggest
             k8s_object[8], # memory limit
-            memoryLimit # suggest
+            results["memory_limit"] # suggest
         ]