working on configuring a k3s cluster

ansible-5/roles/prod.k3s/defaults/main.yml

@@ -3,13 +3,25 @@
 
 fstab:
   gluster:
-    - name: jenkins
-      path: "/var/lib/jenkins"
-      state: mounted
+#    - name: jenkins
+#      path: "/var/lib/jenkins"
+#      state: mounted
     - name: gitea
       path: "/var/lib/gitea"
       state: present
     - name: vmshares
-      path: "/opt/shared glusterfs"
+      path: "/opt/shared"
       state: present
 
+helm:
+  repos:
+    - name: twuni
+      repo_url: https://helm.twun.io
+    - name: jetstack
+      repo_url: https://charts.jetstack.io
+    - name: gitea-charts
+      repo_url: https://dl.gitea.io/charts/
+    - name: bitnami
+      repo_url: https://charts.bitnami.com/bitnami
+    - name: cetic
+      repo_url: https://cetic.github.io/helm-charts

ansible-5/roles/prod.k3s/files/config.yaml

@@ -0,0 +1,7 @@
+write-kubeconfig-mode: 644
+#disable:
+#  - traefik
+#token: "secret"
+#node-ip: 10.0.10.22,2a05:d012:c6f:4655:d73c:c825:a184:1b75
+#cluster-cidr: 10.42.0.0/16,2001:cafe:42:0::/56
+#service-cidr: 10.43.0.0/16,2001:cafe:42:1::/112

ansible-5/roles/prod.k3s/files/manifests/graphana.helm.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: grafana
+  namespace: kube-system
+spec:
+  chart: stable/grafana
+  targetNamespace: monitoring
+  set:
+    adminPassword: "NotVerySafePassword"
+  valuesContent: |-
+    image:
+      tag: master
+    env:
+      GF_EXPLORE_ENABLED: true
+    adminUser: admin
+    sidecar:
+      datasources:
+        enabled: true

ansible-5/roles/prod.k3s/files/manifests/mariadb.helm.yaml

@@ -0,0 +1,26 @@
+#https://github.com/bitnami/charts/tree/master/bitnami/mariadb
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: mariadb
+  namespace: kube-system
+spec:
+  chart: bitnami/mariadb
+  targetNamespace: persistence
+  set:
+    replicaCount: 1
+  valuesContent: |-
+    image:
+      tag: 10.5
+    auth:
+      rootPassword: "aifuoqibcqobcqb3"
+    ingress:
+      className: traefik
+      hosts: sql.xai-corp.net
+    extraVolumeMounts:
+      - name: mysql
+        mountPath: /var/lib/mysql
+    extraVolumes:
+      - name: mysql
+        hostPath:
+          path: /opt/mariadb/data

ansible-5/roles/prod.k3s/files/manifests/note.txt

@@ -0,0 +1,3 @@
+We can add things to the cluster by adding charts to the mainifests folder. These could be k8s resource definitions or helm charts
+
+Are components removed if the chart is removed? - no

ansible-5/roles/prod.k3s/files/manifests/registry.helm.yaml

@@ -0,0 +1,46 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: dkregistry
+  namespace: kube-system
+spec:
+  chart: twuni/docker-registry
+  targetNamespace: dev-tools
+  set:
+    adminPassword: "NotVerySafePassword"
+    replicaCount: 1
+  valuesContent: |-
+    image:
+      tag: 2.7.1
+    metrics:
+      enabled: true
+    ingress:
+      className: traefik
+      hosts: dkregistry.xai-corp.net
+    extraVolumeMounts:
+      - name: registry-data
+        mountPath: /var/lib/registry-data
+      - name: registry-auth
+        mountPath: /auth
+      - name: registry-auth
+        mountPath: /etc/docker/registry/config.yml
+      - name: letsencrypt-data
+        mountPath: /certs
+      - name: letsencrypt-config
+        mountPath: /etc/letsencrypt
+    extraVolumes:
+      - name: registry-data
+        hostPath:
+          path: /opt/shared/dkregistry/data
+      - name: registry-auth
+        hostPath:
+          path: /opt/shared/dkregistry/auth
+      - name: registry-config
+        hostPath:
+          path: /opt/shared/dkregistry/config/config.yml
+      - name: letsencrypt-data
+        hostPath:
+          path: /opt/shared/letsencrypt-2
+      - name: letsencrypt-config
+        hostPath:
+          path: /opt/shared/letsencrypt-2

ansible-5/roles/prod.k3s/tasks/add_repos.yml

@@ -0,0 +1,24 @@
+---
+
+- name: Add repository locally
+  local_action:
+    module: kubernetes.core.helm_repository
+    name: "{{ item.name }}"
+    repo_url: "{{ item.repo_url }}"
+  loop: "{{ helm.repos }}"
+
+- name: Add repository to cluster
+  kubernetes.core.helm_repository:
+    name: "{{ item.name }}"
+    repo_url: "{{ item.repo_url }}"
+  loop: "{{ helm.repos }}"
+  become: true
+
+- name: Separately update the repository cache
+  kubernetes.core.helm:
+    kubeconfig_path: "/etc/rancher/k3s/k3s.yaml"
+    name: dummy
+    namespace: kube-system
+    state: absent
+    update_repo_cache: true
+  become: true

ansible-5/roles/prod.k3s/tasks/main.yml

@@ -11,9 +11,99 @@
     opts: "direct-io-mode=disable,_netdev,x-systemd.automount 0 0"
     state: "{{item.state}}"
   with_items: "{{fstab.gluster}}"
+  become: true
 
 # provision docker image registry
+- include_tasks: add_repos.yml
+
+# https://artifacthub.io/packages/helm/twuni/docker-registry
+- name: Deploy latest version of docker-registry in dev-tools namespace
+  local_action:
+    module: kubernetes.core.helm
+    name: dkregistry
+    chart_ref: twuni/docker-registry
+    release_namespace: dev-tools
+    create_namespace: True
+    values:
+      replicaCount: 1
+      ingress:
+        enabled: true
+        hosts:
+          - dkregistry.xai-corp.net
+        className: traefik
+      secrets.htpassword: me1
+      extraVolumeMounts:
+        - name: registry-data
+          mountPath: /var/lib/registry-data
+        - name: registry-auth
+          mountPath: /auth
+        - name: registry-auth
+          mountPath: /etc/docker/registry/
+        - name: letsencrypt-data
+          mountPath: /certs
+        - name: letsencrypt-config
+          mountPath: /etc/letsencrypt
+      extraVolumes:
+        - name: registry-data
+          hostPath:
+            path: /opt/shared/dkregistry/data
+        - name: registry-auth
+          hostPath:
+            path: /opt/shared/dkregistry/auth
+        - name: registry-config
+          hostPath:
+            path: /opt/shared/dkregistry/config/
+        - name: letsencrypt-data
+          hostPath:
+            path: /opt/shared/letsencrypt-2
+        - name: letsencrypt-config
+          hostPath:
+            path: /opt/shared/letsencrypt-2
+#      extraEnvVars:
+#        - name: REGISTRY_HTTP_SECRET
+#          value: aabuioqlwlcpp2
+#        - name: REGISTRY_HTTP_TLS_CERTIFICATE
+#          value: /certs/live/xai-corp.net/fullchain.pem
+#        - name: REGISTRY_HTTP_TLS_KEY
+#          value: /certs/live/xai-corp.net/privkey.pem
+
+
+# k3s config file
+#- name: Copy k3s config file to /etc/rancher/k3s/config.yaml
+#  ansible.builtin.copy:
+#    src: config.yaml
+#    dest: /etc/rancher/k3s/config.yaml
+#  become: true
 
 # provision gitea
 
 # provision argoCD
+
+# provision graphana
+- name: Copy manifest for graphana
+  ansible.builtin.copy:
+    src: manifests/graphana.helm.yaml
+    dest: /var/lib/rancher/k3s/server/manifests/graphana.helm.yaml
+  become: true
+
+- name: Copy manifest for docker registry
+  ansible.builtin.copy:
+    src: manifests/registry.helm.yaml
+    dest: /var/lib/rancher/k3s/server/manifests/dkregistry.helm.yaml
+  become: true
+
+- name: Copy manifest for mariadb
+  ansible.builtin.copy:
+    src: manifests/mariadb.helm.yaml
+    dest: /var/lib/rancher/k3s/server/manifests/mariadb.helm.yaml
+  become: true
+
+- name: remove manifests
+  ansible.builtin.file:
+    state: absent
+    path: "{{ item }}"
+  loop:
+    - /var/lib/rancher/k3s/server/manifests/graphana.helm.yaml
+    - /var/lib/rancher/k3s/server/manifests/dkregistry.helm.yaml
+    - /var/lib/rancher/k3s/server/manifests/mariadb.helm.yaml
+  become: true