Add kibana instance, fix ltm, add ltm access and service to remember

README.md

@@ -19,8 +19,8 @@ To build images locally one can use the `amygdala-helper.sh` script with the bui
 |-> remember
 |-> stt
 |-> tts
---> vis
-````
+\-> vis
+```
 
 __Note:__ The script assumes the usage of the minikube kubernetes infrastructure and relies on the `minikube cache add/delete` commands. The first time the deletion of the image in the cache will fail as it does not yet exist. This is no problem an can be ignored.
 

amy

+#! /bin/bash
+
+SERVICES=(cnc ltm memorize remember stt tts vis)
+
+function forall_or_one {
+    case "$2" in
+        all )
+            for service in "${SERVICES[@]}";
+            do
+                $1 "$service"
+            done;;
+        * )
+            if [ -f "../$2/deployment/dockerfile" ];
+            then
+                $1 "$3"
+            else
+                echo Unknown image to build;
+            fi
+    esac;
+}
+
+function restart {
+    forall_or_one restart_service "$1" "$2"
+}
+
+function start {
+    minikube start;
+    minikube addons enable registry;
+    kubectl port-forward --namespace kube-system service/;registry 5000:80 > /dev/null & disown || true;
+    istioctl dashboard kiali > /dev/null & disown || true
+    minikube tunnel > /dev/null & disown || true
+}
+
+function apply {
+    istioctl install --set profile=demo -y;
+    kubectl apply -f samples/addons;
+    kubectl rollout status deployment/kiali -n istio-system;
+    kubectl create -f elastic_search/elastic-resources.yaml;
+    kubectl apply -f elastic_search/elastic-operator.yaml;
+    helm install amygdala . --values values.yaml;
+}
+
+
+function delete {
+    helm uninstall amygdala;
+    kubectl delete -f elastic_search/elastic-operator.yaml;
+    kubectl delete -f elastic_search/elastic-resources.yaml;
+    kubectl delete namespace amygdala;
+}
+
+function stop {
+    minikube stop;
+}
+
+function amy-kill {
+    minikube delete;
+}
+
+for subcommand in ./amy.d/*; do
+   source "$subcommand"
+done
+
+function amy {
+    case "$1" in
+        connect )
+            amy-connect;;
+        install )
+            amy-install "$@";;
+        start|setup )
+            start;;
+        up )
+            echo "WARNING: depricated; use $0 apply" "$@";
+            apply;;
+        apply )
+            apply;;
+        build )
+            echo "WARNING: depricated; use $0 service" "$@";
+            amy-build "$2";;
+        service|services )
+            amy-service "$@";;
+        delete|down )
+            delete;;
+        teardown )
+            teardown;;
+        kill )
+            amy-kill;;
+        get )
+            amy-get "$@";;
+        * )
+            echo Unknown Option;;
+    esac
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    amy "$@"
+fi

amy.d/amy-build.sh

+#! /bin/bash
+
+OCI_CLIENT="${OCI_CLIENT:-podman}"
+BASE_TAG="localhost:5000/amygdala"
+
+
+function build_service {
+  tag="$BASE_TAG/$1/$1";
+  $OCI_CLIENT build -t "$tag" -f "../$1/deployment/dockerfile" "../$1/" || exit;
+  $OCI_CLIENT push "$tag";
+}
+
+function amy-build {
+  forall_or_one build_service "$1" "$2"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    amy-build "$@"
+fi

amy.d/amy-connect.sh

+#! /bin/bash
+
+SERVER_IP="${SERVER_IP:-localhost}"
+SERVER_PORT="${SERVER_PORT:-2222}"
+
+AMY_DOMAIN="${AMY_DOMAIN=-amygdala.local}"
+
+function amy-connect {
+    ssh \
+        -L "8443:$AMY_DOMAIN:443"       \
+        -L "5601:ltm.$AMY_DOMAIN:80"    \
+        -L "5000:localhost:5000"        \
+        "$SERVER_IP" -p "$SERVER_PORT"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    amy-connect "$@"
+fi

amy.d/amy-get.sh

+#! /bin/bash
+
+function creds {
+    echo -e "{\n  username=\"elastic\",\n  password=\"$(kubectl -n amygdala get secret ltm-database-es-elastic-user -o go-template='{{.data.elastic|base64decode}}')\"\n}";
+}
+
+function creds {
+    echo -e "{\n  tls_cert=\"$(kubectl -n amygdala get secret ltm-database-es-http-certs-public -o go-template='{{index .data "tls.crt"|base64decode}}')\"\n}";
+}
+
+function amy-get {
+    case "$1" in
+        creds )
+            creds;;
+        tls_cert )
+            certs;;
+        * )
+            kubectl -n amygdala get "$1"
+    esac;
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    amy-get "$@"
+fi
\ No newline at end of file

amy.d/amy-install.sh

+#! /bin/bash
+
+function amy-install {
+    case "$1" in
+        istio )
+            echo "downloading istio";
+            curl -L https://istio.io/downloadIstio | sh -;
+            cd "istio-1.13.3" || exit 1;
+            echo 'export PATH=$PWD/bin:$PATH' >> ~/.bashrc;
+            source ~/.bashrc;;
+        * )
+            echo "not implemented"
+    esac;
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    amy-install "$@"
+fi
\ No newline at end of file

amy.d/amy-service.sh

+#! /bin/bash
+
+function service_restart {
+  kubectl -n amygdala rollout restart "deployment/$1-dep";
+}
+
+function amy-service {
+    case "$1" in
+        build )
+            amy-build "$2";;
+        restart )
+            service_restart "$@";;
+    esac
+}
\ No newline at end of file

amygdala-helper.sh

-#!/bin/bash
-
-OCI_CLIENT="${OCI_CLIENT:-podman}"
-BASE_TAG="localhost:5000/amygdala"
-SERVICES=(cnc ltm memorize remember stt tts vis)
-
-function build {
-  tag="$BASE_TAG/$1/$1";
-  $OCI_CLIENT build -t "$tag" -f "../$1/deployment/dockerfile" "../$1/" || exit;
-  $OCI_CLIENT push "$tag";
-}
-
-function build_service {
-  if [ -f "../$1/deployment/dockerfile" ];
-  then
-    build "$1"
-  else
-      echo Unknown image to build;
-  fi
-}
-
-case "$1" in
-  setup )
-    minikube start;
-    minikube addons enable ingress;
-    minikube addons enable registry;
-    kubectl port-forward --namespace kube-system service/registry 5000:80 & disown || true;;
-	up )
-		helm install amygdala . --values values.yaml;;
-  restart )
-    case "$2" in
-      all )
-        for service in "${SERVICES[@]}";
-        do
-          kubectl -n amygdala rollout restart "deployment/$service-dep"
-        done;;
-      * )
-        kubectl -n amygdala rollout restart "deployment/$2-dep";;
-    esac;;
-	down )
-		helm uninstall amygdala;
-    kubectl delete namespace amygdala;;
-  build )
-    case "$2" in
-      all )
-        for service in "${SERVICES[@]}";
-        do
-          build_service "$service";
-        done;;
-      * )
-        build_service "$2";;
-    esac;;
-  teardown )
-    minikube stop;;
-  delete )
-    minikube delete;;
-	* )
-		echo Unknown Option;;
-esac

database-helper.sh

-#!/bin/bash
-
-case "$1" in
-	up )
-		kubectl create -f elastic_search/elastic-resources.yaml;
-		kubectl apply -f elastic_search/elastic-operator.yaml;;
-	down )
-		kubectl delete -f elastic_search/elastic-operator.yaml;
-		kubectl delete -f elastic_search/elastic-resources.yaml;;
-	creds )
-		echo -e "{\n  username=\"elastic\",\n  password=\"$(kubectl -n amygdala get secret ltm-database-es-elastic-user -o go-template='{{.data.elastic | base64decode}}')\"\n}";;
-  tls_cert )
-		echo -e "{\n  tls_cert=\"$(kubectl -n amygdala get secret ltm-database-es-http-certs-public -o go-template='{{index .data "tls.crt" | base64decode}}')\"\n}";;
-	* )
-		echo Unknown Option;;	
-esac

templates/amygdala.yaml

@@ -7,3 +7,30 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: amygdala
     meta.helm.sh/release-name: amygdala
+
+---
+
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: amygdala-gateway
+  namespace: amygdala
+spec:
+  selector:
+    istio: ingressgateway
+  servers:
+  - port:
+      number: 80
+      name: kibana
+      protocol: HTTP
+    hosts:
+    - ltm.amygdala.local
+  - port:
+      number: 443
+      name: https
+      protocol: HTTPS
+    hosts:
+    - amygdala.local
+    tls:
+      mode: SIMPLE
+      credentialName: amygdala-tls-config

templates/ltm-database.yaml

@@ -23,7 +23,7 @@ metadata:
   name: ltm-database
   namespace: amygdala
 spec:
-  version: 8.1.1
+  version: 8.2.0
   nodeSets:
   # This configuration is for testing only
   - name: node

templates/ltm-kibana.yaml

+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: ltm-kibana-virt-svc
+  namespace: amygdala
+spec:
+  hosts:
+  - ltm.amygdala.local
+  gateways:
+  - amygdala-gateway
+  http:
+  - match:
+    - uri:
+        prefix: /
+    route:
+    - destination:
+        host: ltm-kibana-kb-http
+        port:
+          number: 5601
+
+---
+
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: elasticsearch-kibana-nwp
+  namespace: amygdala
+spec:
+  podSelector:
+    matchLabels:
+      network/elasticsearch-kibana: "true"
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+      - podSelector:
+          matchLabels:
+            network-access/elasticsearch-kibana: "true"
+
+---
+
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: ltm-kibana
+  namespace: amygdala
+spec:
+  version: 8.2.0
+  count: 1
+  elasticsearchRef:
+    name: "ltm-database"
+  podTemplate:
+    metadata:
+      labels:
+        app.kubernetes.io/name: node
+        app.kubernetes.io/managed-by: Helm
+        network/elasticsearch-kibana: "true"
+        network-access/elasticsearch-cluster: "true"
+  http:
+    tls:
+      selfSignedCertificate:
+        disabled: true
+  config:
+     elasticsearch.requestHeadersWhitelist:
+     - authorization
+

templates/ltm.yaml

@@ -31,9 +31,9 @@ metadata:
     meta.helm.sh/release-name: amygdala
 spec:
   ports:
-    - name: "ltm"
-      port: 8080
-      targetPort: 8080
+    - name: "grpc"
+      port: 50051
+      targetPort: 50051
   selector:
     app.kubernetes.io/name: ltm-pod
 
@@ -73,6 +73,8 @@ spec:
     metadata:
       labels:
         network/ltm: "true"
+        network-access/memorize: "true"
+        network-access/remember: "true"
         network-access/elastic-cluster: "true"
         app.kubernetes.io/name: ltm-pod
         app.kubernetes.io/version: "1.0.0"
@@ -88,7 +90,7 @@ spec:
           imagePullPolicy: {{ . }}
           {{- end }}
           ports:
-            - containerPort: 8080
+            - containerPort: 50051
           env:
             - name: ELASTICSEARCH_PASSWORD
               valueFrom:

templates/memorize.yaml

@@ -17,32 +17,27 @@ spec:
 
 ---
 
-apiVersion: networking.k8s.io/v1
-kind: Ingress
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
 metadata:
-  name: memorize-ing
+  name: memorize-virt-svc
   namespace: amygdala
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /memorize
   labels:
-    app.kubernetes.io/name: memorize-ing
-    app.kubernetes.io/version: "1.0.0"
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: frontend
-    app.kubernetes.io/part-of: amygdala
-    meta.helm.sh/release-name: amygdala
+    network-access/vis: "true"
 spec:
-  rules:
-  - host: amygdala
-    http:
-      paths:
-      - path: /memorize
-        pathType: Prefix
-        backend:
-          service:
-            name: memorize-svc
-            port:
-              number: 8080
+  hosts:
+  - amygdala.local
+  gateways:
+  - amygdala-gateway
+  http:
+  - match:
+    - uri:
+        prefix: /memorize
+    route:
+    - destination:
+        host: memorize-svc
+        port:
+          number: 8080
 
 ---
 
@@ -104,6 +99,8 @@ spec:
           env:
             - name: STT_SERVER_HOST
               value: stt-svc:50051
+            - name: LTM_SERVER_HOST
+              value: ltm-svc:50051
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}

templates/remember.yaml

@@ -17,32 +17,27 @@ spec:
 
 ---
 
-apiVersion: networking.k8s.io/v1
-kind: Ingress
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
 metadata:
-  name: remember-ing
+  name: remember-virt-svc
   namespace: amygdala
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /remember
   labels:
-    app.kubernetes.io/name: remember-ing
-    app.kubernetes.io/version: "1.0.0"
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: frontend
-    app.kubernetes.io/part-of: amygdala
-    meta.helm.sh/release-name: amygdala
+    network-access/vis: "true"
 spec:
-  rules:
-  - host: amygdala
-    http:
-      paths:
-      - path: /remember
-        pathType: Prefix
-        backend:
-          service:
-            name: remember-svc
-            port:
-              number: 8080
+  hosts:
+  - amygdala.local
+  gateways:
+  - amygdala-gateway
+  http:
+  - match:
+    - uri:
+        prefix: /remember
+    route:
+    - destination:
+        host: remember-svc
+        port:
+          number: 8080
 
 ---
 
@@ -110,6 +105,8 @@ spec:
               value: tts-svc:50051
             - name: CNC_SERVER_HOST
               value: cnc-svc:50051
+            - name: LTM_SERVER_HOST
+              value: ltm-svc:50051
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}

templates/vis.yaml

@@ -17,32 +17,27 @@ spec:
 
 ---
 
-apiVersion: networking.k8s.io/v1
-kind: Ingress
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
 metadata:
-  name: vis-ing
+  name: vis-virt-svc
   namespace: amygdala
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
   labels:
-    app.kubernetes.io/name: vis-ing
-    app.kubernetes.io/version: "1.0.0"
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/component: frontend
-    app.kubernetes.io/part-of: amygdala
-    meta.helm.sh/release-name: amygdala
+    network-access/vis: "true"
 spec:
-  rules:
-  - host: amygdala
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: vis-svc
-            port:
-              number: 8080
+  hosts:
+  - amygdala.local
+  gateways:
+  - amygdala-gateway
+  http:
+  - match:
+    - uri:
+        prefix: /
+    route:
+    - destination:
+        host: vis-svc
+        port:
+          number: 8080
 
 ---
 
@@ -60,7 +55,7 @@ metadata:
     meta.helm.sh/release-name: amygdala
 spec:
   ports:
-    - name: "web-access-port"
+    - name: http
       port: 8080
       targetPort: 80
   selector: