Add additional external connectivity Checks to kconmon

Dockerfile

@@ -18,7 +18,11 @@ RUN addgroup nonroot && \
     adduser -D nonroot -G nonroot && \
     chown nonroot:nonroot /app
 
+RUN apk update && \
+    apk add --no-cache iputils
+
 USER nonroot
+
 RUN mkdir -p /home/nonroot/.npm
 VOLUME /home/nonroot/.npm
 COPY package.json ./

Gruntfile.js

@@ -35,7 +35,8 @@ const mochaConfig = {
 
 const eslintConfig = {
   options: {
-    configFile: '.eslintrc.js'
+    configFile: '.eslintrc.js',
+    fix: true
   },
   target: config.targets.ts
 }

helmfile/charts/kconmon/values.yaml

@@ -1,5 +1,6 @@
 docker:
   image: stono/kconmon
+  tag: latest
 
 # Should we run an initContainer that enables core tcp connection setting tweaks
 # Note: This requires kernel 4+
@@ -38,6 +39,26 @@ config:
       - www.google.com
       - kubernetes.default.svc.cluster.local
 
+# ICMP Test configuration
+  icmp:
+    enable: true
+    interval: 5000
+    count: 2
+    timeout: 5
+    hosts:
+      - www.telekom.de
+      - www.google.com
+      - 8.8.4.4
+
+# Custom HTTP tests configuration
+  custom_http:
+    enable: true
+    interval: 5000
+    timeout: 1000
+    hosts:
+      - www.telekom.de
+      - www.google.de
+
 resources:
   agent:
     # This will scale with the number of nodes in your cluster at loosely 1m per node

lib/apps/agent/metrics.ts

@@ -1,14 +1,23 @@
 export interface IMetrics {
   handleTCPTestResult(result: ITCPTestResult)
+  handleCustomHTTPTestResult(result: ICustomHTTPTestResult)
   handleUDPTestResult(result: IUDPTestResult)
   handleDNSTestResult(result: IDNSTestResult)
+  handleICMPTestResult(result: IICMPTestResult)
   resetTCPTestResults()
+  resetCustomHTTPTestResults()
   resetUDPTestResults()
   toString()
 }
 
 import * as client from 'prom-client'
-import { IUDPTestResult, IDNSTestResult, ITCPTestResult } from 'lib/tester'
+import {
+  IICMPTestResult,
+  IUDPTestResult,
+  IDNSTestResult,
+  ITCPTestResult,
+  ICustomHTTPTestResult
+} from 'lib/tester'
 import { IConfig } from 'lib/config'
 
 export default class Metrics implements IMetrics {
@@ -24,6 +33,16 @@ export default class Metrics implements IMetrics {
   private DNS: client.Counter<string>
   private DNSDuration: client.Gauge<string>
 
+  private ICMP: client.Counter<string>
+  private ICMPDuration: client.Gauge<string>
+  private ICMPAverage: client.Gauge<string>
+  private ICMPStddv: client.Gauge<string>
+  private ICMPLoss: client.Gauge<string>
+
+  private CustomHTTP: client.Counter<string>
+  private CustomHTTPDuration: client.Gauge<string>
+  private CustomHTTPConnect: client.Gauge<string>
+
   constructor(config: IConfig) {
     client.register.clear()
     this.TCPConnect = new client.Gauge<string>({
@@ -68,6 +87,36 @@ export default class Metrics implements IMetrics {
       name: `${config.metricsPrefix}_dns_duration_milliseconds`
     })
 
+    this.ICMP = new client.Counter<string>({
+      help: 'ICMP Test Results',
+      labelNames: ['source', 'source_zone', 'host', 'result'],
+      name: `${config.metricsPrefix}_icmp_results_total`
+    })
+
+    this.ICMPDuration = new client.Gauge<string>({
+      help: 'Total time taken to complete the ICMP test',
+      labelNames: ['source', 'source_zone', 'host'],
+      name: `${config.metricsPrefix}_icmp_duration_milliseconds`
+    })
+
+    this.ICMPAverage = new client.Gauge<string>({
+      help: 'ICMP average packet RTT',
+      labelNames: ['source', 'source_zone', 'host'],
+      name: `${config.metricsPrefix}_icmp_average_rtt_milliseconds`
+    })
+
+    this.ICMPStddv = new client.Gauge<string>({
+      help: 'ICMP standard deviation of RTT',
+      labelNames: ['source', 'source_zone', 'host'],
+      name: `${config.metricsPrefix}_icmp_standard_deviation_rtt_milliseconds`
+    })
+
+    this.ICMPLoss = new client.Gauge<string>({
+      help: 'ICMP packet loss',
+      labelNames: ['source', 'source_zone', 'host'],
+      name: `${config.metricsPrefix}_icmp_packet_loss`
+    })
+
     this.UDP = new client.Counter<string>({
       help: 'UDP Test Results',
       labelNames: [
@@ -91,6 +140,63 @@ export default class Metrics implements IMetrics {
       ],
       name: `${config.metricsPrefix}_tcp_results_total`
     })
+
+    this.CustomHTTP = new client.Counter<string>({
+      help: 'Custom TCP Test Results',
+      labelNames: [
+        'source',
+        'destination',
+        'source_zone',
+        'destination_zone',
+        'result'
+      ],
+      name: `${config.metricsPrefix}_custom_http_results_total`
+    })
+
+    this.CustomHTTPConnect = new client.Gauge<string>({
+      help: 'Time taken to establish the TCP socket for custom test',
+      labelNames: ['source', 'destination', 'source_zone', 'destination_zone'],
+      name: `${config.metricsPrefix}_custom_http_connect_milliseconds`
+    })
+
+    this.CustomHTTPDuration = new client.Gauge<string>({
+      help: 'Total time taken to complete the custom TCP test',
+      labelNames: ['source', 'destination', 'source_zone', 'destination_zone'],
+      name: `${config.metricsPrefix}_custom_http_duration_milliseconds`
+    })
+  }
+
+  public handleICMPTestResult(result: IICMPTestResult): void {
+    const source = result.source.nodeName
+    this.ICMP.labels(
+      source,
+      result.source.zone,
+      result.host,
+      result.result
+    ).inc(1)
+    this.ICMPDuration.labels(
+      result.source.nodeName,
+      result.source.zone,
+      result.host
+    ).set(result.duration)
+
+    this.ICMPAverage.labels(
+      result.source.nodeName,
+      result.source.zone,
+      result.host
+    ).set(result.avg)
+
+    this.ICMPStddv.labels(
+      result.source.nodeName,
+      result.source.zone,
+      result.host
+    ).set(result.stddev)
+
+    this.ICMPLoss.labels(
+      result.source.nodeName,
+      result.source.zone,
+      result.host
+    ).set(result.loss)
   }
 
   public handleDNSTestResult(result: IDNSTestResult): void {
@@ -183,6 +289,44 @@ export default class Metrics implements IMetrics {
     }
   }
 
+  public handleCustomHTTPTestResult(result: ICustomHTTPTestResult): void {
+    const source = result.source.nodeName
+    const destination = result.destination
+    const sourceZone = result.source.zone
+    const destinationZone = result.destination
+    this.CustomHTTP.labels(
+      source,
+      destination,
+      sourceZone,
+      destinationZone,
+      result.result
+    ).inc(1)
+
+    if (result.timings) {
+      this.CustomHTTPConnect.labels(
+        source,
+        destination,
+        sourceZone,
+        destinationZone
+      ).set(
+        ((result.timings.connect ||
+          result.timings.socket ||
+          result.timings.start) - result.timings.start) as number
+      )
+      this.CustomHTTPDuration.labels(
+        source,
+        destination,
+        sourceZone,
+        destinationZone
+      ).set(result.timings.phases.total as number)
+    }
+  }
+
+  public resetCustomHTTPTestResults() {
+    this.CustomHTTPConnect.reset()
+    this.CustomHTTPDuration.reset()
+  }
+
   public toString(): string {
     return client.register.metrics()
   }

lib/config/index.ts

@@ -28,6 +28,19 @@ interface ITestConfiguration {
     interval: number
     hosts: string[]
   }
+  icmp: {
+    enable: boolean
+    interval: number
+    count: number
+    timeout: number
+    hosts: string[]
+  }
+  custom_http: {
+    enable: boolean
+    interval: number
+    timeout: number
+    hosts: string[]
+  }
 }
 
 export interface IConfig {
@@ -60,7 +73,13 @@ export class Config implements IConfig {
   public readonly testConfig: ITestConfiguration = {
     tcp: getEnv('tcp', { interval: 5000, timeout: 1000 }),
     udp: getEnv('udp', { interval: 5000, timeout: 250, packets: 10 }),
-    dns: getEnv('dns', { interval: 5000, hosts: [] })
+    dns: getEnv('dns', { interval: 5000, hosts: [] }),
+    icmp: getEnv('icmp', { interval: 5000, hosts: [] }),
+    custom_http: getEnv('custom_http', {
+      interval: 5000,
+      timeout: 1000,
+      hosts: []
+    })
   }
 }