working on configuring a k3s cluster

.idea/misc.xml

@@ -3,9 +3,4 @@
   <component name="JavaScriptSettings">
     <option name="languageLevel" value="ES6" />
   </component>
-  <component name="SwUserDefinedSpecifications">
-    <option name="specTypeByUrl">
-      <map />
-    </option>
-  </component>
 </project>

ansible-5/inventory.ini

@@ -14,7 +14,11 @@ cubox-i         ansible_ssh_host=192.168.4.12
 
 [kube]
 home            ansible_ssh_host=192.168.4.11
+;cubox-m         ansible_ssh_host=192.168.4.15
 
 ;[desktop]
 ;richard-desktop ansible_connection=local
 
+[k0s]
+cubox-m     ansible_ssh_host=192.168.4.15
+

ansible-5/playbooks/bootstrap.yaml

@@ -1,7 +1,7 @@
 ---
 - name: bootstrap
-  hosts: all
+  hosts: cubox-m
-  remote_user: richard
+  remote_user: ansible
   gather_facts: false
 #  become: true
 
@@ -14,6 +14,12 @@
     - name: Creates .ssh directory
       file: path=~/.ssh state=directory mode=700
 
+    - name: remove debian user if it exists
+      command: userdel -rf debian
+      become: true
+      args:
+        removes: /home/debian/.bashrc
+
 #    - name: remove ubuntu user if it exists
 #      command: userdel -rf ubuntu
 #      args:
@@ -41,18 +47,6 @@
         groups:
           - sudo
 
-    - name: Add the user 'richard'
-      become: true
-      ansible.builtin.user:
-        name: richard
-        state: present
-        shell: /bin/bash
-        create_home: yes
-        password: "$6$yNKLUxX0$lxy/jaJI7cKCq5j.KondUalu9r96gUeRR//5qciZ/RX9z9PGSpbU9j7OsxaOzqV5uLeQ9ouIe8quo/2YqKE46/"
-        uid: "1000"
-        groups:
-          - sudo
-
     - name: Add the authorized key for 'ansible'
       become: true
       ansible.posix.authorized_key:
@@ -77,6 +71,27 @@
            62666132613033633733336434373161316664626531336363306664373131303937383066363066
            636534343631376365633666316534663932
 
+    - name: add ansible to sudoers
+      become: true
+      lineinfile:
+        dest: /etc/sudoers
+        state: present
+        regexp: '^ansible ALL='
+        line: 'ansible ALL=(ALL) NOPASSWD: ALL'
+        validate: 'visudo -cf %s'
+
+    - name: Add the user 'richard'
+      become: true
+      ansible.builtin.user:
+        name: richard
+        state: present
+        shell: /bin/bash
+        create_home: yes
+        password: "$6$yNKLUxX0$lxy/jaJI7cKCq5j.KondUalu9r96gUeRR//5qciZ/RX9z9PGSpbU9j7OsxaOzqV5uLeQ9ouIe8quo/2YqKE46/"
+        uid: "1000"
+        groups:
+          - sudo
+
     - name: Add the authorized key for 'richard'
       become: true
       ansible.posix.authorized_key:
@@ -87,11 +102,3 @@
       with_file:
         - '/home/richard/.ssh/id_rsa.pub'
 
-    - name: add ansible to sudoers
-      become: true
-      lineinfile:
-        dest: /etc/sudoers
-        state: present
-        regexp: '^ansible ALL='
-        line: 'ansible ALL=(ALL) NOPASSWD: ALL'
-        validate: 'visudo -cf %s'

ansible-5/playbooks/k0s.yaml

@@ -0,0 +1,10 @@
+---
+- name: ping
+  hosts: cubox-m
+  gather_facts: true
+
+  roles:
+    - role: k0s
+      become: true
+#    - role: prod.k3s
+#      become: true

ansible-5/playbooks/kube.yaml

@@ -7,4 +7,4 @@
     - role: k3s
       become: true
     - role: prod.k3s
-      become: true
+#      become: true

ansible-5/roles/k0s/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+k0s_version: v1.22.4+k0s.1
+k0s_binary_dest: /usr/local/bin/k0s
+
+k0s_config_dir: /etc/k0s
+k0s_data_dir: /var/lib/k0s
+k0s_libexec_dir: /usr/libexec/k0s/
+k0s_use_custom_config: false
+
+artifacts_dir: "{{ inventory_dir }}/artifacts"

ansible-5/roles/k0s/tasks/dir_config.yml

@@ -0,0 +1,40 @@
+
+
+---
+
+- name: Create k0s Directories
+  become: true
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+  loop:
+    - "{{ k0s_config_dir }}"
+    - "{{ k0s_data_dir }}"
+    - "{{ k0s_libexec_dir }}"
+
+- name: Write the custom k0s config file
+  template:
+    src: k0s.yaml.j2
+    dest: "{{ k0s_config_dir }}/k0s.yaml"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: 0600
+  when: k0s_use_custom_config
+
+- name: Generate default k0s config file
+  become: true
+  block:
+    - name: Create default k0s config
+      register: default_k0s_config
+      command: k0s default-config > {{ k0s_config_dir }}/k0s.yaml
+    - name: Store default k0f config
+      copy:
+        dest: "{{ k0s_config_dir }}/k0s.yaml"
+        content: "{{ default_k0s_config.stdout }}"
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+        mode: 0600
+  when: not k0s_use_custom_config

ansible-5/roles/k0s/tasks/download.yml

@@ -0,0 +1,22 @@
+---
+
+- name: Download k0s binary k0s-{{ k0s_version }}-amd64
+  get_url:
+    url: https://github.com/k0sproject/k0s/releases/download/{{ k0s_version }}/k0s-{{ k0s_version }}-amd64
+    dest: "{{ k0s_binary_dest }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: 0755
+  when: ansible_facts.architecture == "x86_64"
+
+- name: Download k0s binary k0s-{{ k0s_version }}-arm64
+  get_url:
+    url: https://github.com/k0sproject/k0s/releases/download/{{ k0s_version }}/k0s-{{ k0s_version }}-arm64
+    dest: "{{ k0s_binary_dest }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: 0755
+  when:
+    - ( ansible_facts.architecture is search("arm") and
+      ansible_facts.userspace_bits == "64" ) or
+      ansible_facts.architecture is search("aarch64")

ansible-5/roles/k0s/tasks/initial_controller.yml

@@ -0,0 +1,90 @@
+
+
+---
+
+- name: Create k0s initial controller service with install command
+  register: install_initial_controller_cmd
+  command: k0s install controller --config {{ k0s_config_dir }}/k0s.yaml {{ extra_args | default(omit) }}
+  changed_when: install_initial_controller_cmd | length > 0
+
+- name: Setup custom environment variables for systemd unit
+  include_role:
+    name: env_setup
+
+- name: Enable and check k0s service
+  systemd:
+    name: k0scontroller
+    daemon_reload: yes
+    state: restarted
+    enabled: yes
+
+- name: Wait for k8s apiserver
+  wait_for:
+    host: localhost
+    port: 6443
+    delay: 15
+    timeout: 180
+
+- name: Create worker join token
+  register: worker_join_token
+  command: k0s token create --role worker  --config {{ k0s_config_dir }}/k0s.yaml
+  changed_when: worker_join_token | length > 0
+
+- name: Store worker join token
+  set_fact:
+    join_token_worker: "{{ worker_join_token.stdout }}"
+    cacheable: yes
+
+- name: Add k0s worker token to dummy host
+  add_host:
+    name: "worker_token_holder"
+    token: "{{ worker_join_token.stdout }}"
+
+- name: Print worker token
+  debug:
+    msg: "k0s worker join token is: {{ worker_join_token.stdout }}"
+
+- name: Create controller join token
+  register: controller_join_token
+  command: k0s token create --role controller --config {{ k0s_config_dir }}/k0s.yaml
+  changed_when: controller_join_token | length > 0
+
+- name: Store controller join token
+  set_fact:
+    join_token_controller: "{{ controller_join_token.stdout }}"
+    cacheable: yes
+
+- name: Add k0s controller token to dummy host
+  add_host:
+    name: "controller_token_holder"
+    token: "{{ controller_join_token.stdout }}"
+
+- name: Print controller token
+  debug:
+    msg: "k0s controller join token is: {{ controller_join_token.stdout }}"
+
+- name: Copy config file to user home directory
+  copy:
+    src: "{{ k0s_data_dir }}/pki/admin.conf"
+    dest: ~{{ ansible_user }}/k0s-kubeconfig.yml
+    remote_src: yes
+    owner: "{{ ansible_user }}"
+    mode: 0644
+
+- name: Set controller IP in kubeconfig
+  replace:
+    path: ~{{ ansible_user }}/k0s-kubeconfig.yml
+    regexp: 'localhost'
+    replace: "{{ ansible_host }}"
+
+- name: Copy kubeconfig
+  fetch:
+    src: "~{{ ansible_user }}/k0s-kubeconfig.yml"
+    dest: "{{ artifacts_dir }}/k0s-kubeconfig.yml"
+    flat: yes
+    validate_checksum: no
+  become: no
+
+- name: "print kubeconfig command"
+  debug:
+    msg: "To use Cluster: export KUBECONFIG={{ artifacts_dir }}/k0s-kubeconfig.yml"

ansible-5/roles/k0s/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+#main install of k0s
+
+- include_tasks: download.yml
+- include_tasks: dir_config.yml
+

ansible-5/roles/k0s/templates/k0s.yaml.j2

@@ -0,0 +1,76 @@
+apiVersion: k0s.k0sproject.io/v1beta1
+kind: ClusterConfig
+metadata:
+  creationTimestamp: null
+  name: k0s
+spec:
+  api:
+    address: 192.168.4.15
+    k0sApiPort: 9443
+    port: 6443
+    sans:
+    - 192.168.4.15
+  controllerManager: {}
+  images:
+    calico:
+      cni:
+        image: docker.io/calico/cni
+        version: v3.18.1
+      kubecontrollers:
+        image: docker.io/calico/kube-controllers
+        version: v3.18.1
+      node:
+        image: docker.io/calico/node
+        version: v3.18.1
+    coredns:
+      image: k8s.gcr.io/coredns/coredns
+      version: v1.7.0
+    default_pull_policy: IfNotPresent
+    konnectivity:
+      image: k8s.gcr.io/kas-network-proxy/proxy-agent
+      version: v0.0.25
+    kubeproxy:
+      image: k8s.gcr.io/kube-proxy
+      version: v1.22.4
+    kuberouter:
+      cni:
+        image: docker.io/cloudnativelabs/kube-router
+        version: v1.3.2
+      cniInstaller:
+        image: quay.io/k0sproject/cni-node
+        version: 0.1.0
+    metricsserver:
+      image: k8s.gcr.io/metrics-server/metrics-server
+      version: v0.5.0
+  installConfig:
+    users:
+      etcdUser: etcd
+      kineUser: kube-apiserver
+      konnectivityUser: konnectivity-server
+      kubeAPIserverUser: kube-apiserver
+      kubeSchedulerUser: kube-scheduler
+  konnectivity:
+    adminPort: 8133
+    agentPort: 8132
+  network:
+    calico: null
+    dualStack: {}
+    kubeProxy:
+      mode: iptables
+    kuberouter:
+      autoMTU: true
+      mtu: 0
+      peerRouterASNs: ""
+      peerRouterIPs: ""
+    podCIDR: 10.244.0.0/16
+    provider: kuberouter
+    serviceCIDR: 10.96.0.0/12
+  podSecurityPolicy:
+    defaultPolicy: 00-k0s-privileged
+  scheduler: {}
+  storage:
+    etcd:
+      peerAddress: 192.168.4.15
+    type: etcd
+  telemetry:
+    enabled: true

ansible-5/roles/k3s/tasks/install.yml

@@ -23,7 +23,7 @@
     cmd: sh -s --
     stdin: "{{ k3s_installer.content }}"
 
-- name: Setup bash completion
+#- name: Setup bash completion
-  ansible.builtin.shell:
+#  ansible.builtin.shell:
-    cmd: "kubectl completion bash >/etc/bash_completion.d/kubectl"
+#    cmd: "kubectl completion bash >/etc/bash_completion.d/kubectl"
-    creates: /etc/bash_completion.d/kubectl
+#    creates: /etc/bash_completion.d/kubectl

ansible-5/roles/k3s/tasks/install_helm.yml

@@ -0,0 +1,34 @@
+---
+#$ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+#$ chmod 700 get_helm.sh
+#$ ./get_helm.sh
+
+- name: Install required packages
+  apt:
+    name: "{{ item }}"
+    update_cache: yes
+    cache_valid_time: 3600
+    state: latest
+  with_items:
+    - curl
+    - bash-completion
+
+- name: Fetch helm install script
+  ansible.builtin.uri:
+    url: https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+    return_content: yes
+    creates: /usr/local/bin/helm
+  register: helm_installer
+
+- debug:
+    var: helm_installer
+
+- name: Run installer
+  async: 300
+  poll: 10
+  ansible.builtin.shell:
+    cmd: bash -s --
+    stdin: "{{ helm_installer.content }}"
+#    creates: /usr/local/bin/helm
+  when: helm_installer.changed
+

ansible-5/roles/k3s/tasks/main.yml

@@ -7,9 +7,12 @@
   register: k3s_service
 
 - include_tasks: install.yml
-#  when: not k3s_service.stat.exists
+  when: not k3s_service.stat.exists
 
 - name: Start service k3s, if not started
   ansible.builtin.service:
     name: k3s
     state: started
+
+- include_tasks: install_helm.yml
+

ansible-5/roles/ns.xai-corp.net/templates/xai-corp.net.internal.j2

@@ -23,6 +23,7 @@ garden                  IN A    192.168.4.20
 ; bare metal servers
 home                    IN A    192.168.4.11
 cubox-i                 IN A    192.168.4.12
+cubox-m                 IN A    192.168.4.15
 
 ; virtual machine servers
 home02                  IN A    192.168.4.22

ansible-5/roles/prod.k3s/defaults/main.yml

@@ -3,13 +3,25 @@
 
 fstab:
   gluster:
-    - name: jenkins
+#    - name: jenkins
-      path: "/var/lib/jenkins"
+#      path: "/var/lib/jenkins"
-      state: mounted
+#      state: mounted
     - name: gitea
       path: "/var/lib/gitea"
       state: present
     - name: vmshares
-      path: "/opt/shared glusterfs"
+      path: "/opt/shared"
       state: present
 
+helm:
+  repos:
+    - name: twuni
+      repo_url: https://helm.twun.io
+    - name: jetstack
+      repo_url: https://charts.jetstack.io
+    - name: gitea-charts
+      repo_url: https://dl.gitea.io/charts/
+    - name: bitnami
+      repo_url: https://charts.bitnami.com/bitnami
+    - name: cetic
+      repo_url: https://cetic.github.io/helm-charts

ansible-5/roles/prod.k3s/files/config.yaml

@@ -0,0 +1,7 @@
+write-kubeconfig-mode: 644
+#disable:
+#  - traefik
+#token: "secret"
+#node-ip: 10.0.10.22,2a05:d012:c6f:4655:d73c:c825:a184:1b75
+#cluster-cidr: 10.42.0.0/16,2001:cafe:42:0::/56
+#service-cidr: 10.43.0.0/16,2001:cafe:42:1::/112

ansible-5/roles/prod.k3s/files/manifests/graphana.helm.yaml

@@ -0,0 +1,19 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: grafana
+  namespace: kube-system
+spec:
+  chart: stable/grafana
+  targetNamespace: monitoring
+  set:
+    adminPassword: "NotVerySafePassword"
+  valuesContent: |-
+    image:
+      tag: master
+    env:
+      GF_EXPLORE_ENABLED: true
+    adminUser: admin
+    sidecar:
+      datasources:
+        enabled: true

ansible-5/roles/prod.k3s/files/manifests/mariadb.helm.yaml

@@ -0,0 +1,26 @@
+#https://github.com/bitnami/charts/tree/master/bitnami/mariadb
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: mariadb
+  namespace: kube-system
+spec:
+  chart: bitnami/mariadb
+  targetNamespace: persistence
+  set:
+    replicaCount: 1
+  valuesContent: |-
+    image:
+      tag: 10.5
+    auth:
+      rootPassword: "aifuoqibcqobcqb3"
+    ingress:
+      className: traefik
+      hosts: sql.xai-corp.net
+    extraVolumeMounts:
+      - name: mysql
+        mountPath: /var/lib/mysql
+    extraVolumes:
+      - name: mysql
+        hostPath:
+          path: /opt/mariadb/data

ansible-5/roles/prod.k3s/files/manifests/note.txt

@@ -0,0 +1,3 @@
+We can add things to the cluster by adding charts to the mainifests folder. These could be k8s resource definitions or helm charts
+
+Are components removed if the chart is removed? - no

ansible-5/roles/prod.k3s/files/manifests/registry.helm.yaml

@@ -0,0 +1,46 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: dkregistry
+  namespace: kube-system
+spec:
+  chart: twuni/docker-registry
+  targetNamespace: dev-tools
+  set:
+    adminPassword: "NotVerySafePassword"
+    replicaCount: 1
+  valuesContent: |-
+    image:
+      tag: 2.7.1
+    metrics:
+      enabled: true
+    ingress:
+      className: traefik
+      hosts: dkregistry.xai-corp.net
+    extraVolumeMounts:
+      - name: registry-data
+        mountPath: /var/lib/registry-data
+      - name: registry-auth
+        mountPath: /auth
+      - name: registry-auth
+        mountPath: /etc/docker/registry/config.yml
+      - name: letsencrypt-data
+        mountPath: /certs
+      - name: letsencrypt-config
+        mountPath: /etc/letsencrypt
+    extraVolumes:
+      - name: registry-data
+        hostPath:
+          path: /opt/shared/dkregistry/data
+      - name: registry-auth
+        hostPath:
+          path: /opt/shared/dkregistry/auth
+      - name: registry-config
+        hostPath:
+          path: /opt/shared/dkregistry/config/config.yml
+      - name: letsencrypt-data
+        hostPath:
+          path: /opt/shared/letsencrypt-2
+      - name: letsencrypt-config
+        hostPath:
+          path: /opt/shared/letsencrypt-2

ansible-5/roles/prod.k3s/tasks/add_repos.yml

@@ -0,0 +1,24 @@
+---
+
+- name: Add repository locally
+  local_action:
+    module: kubernetes.core.helm_repository
+    name: "{{ item.name }}"
+    repo_url: "{{ item.repo_url }}"
+  loop: "{{ helm.repos }}"
+
+- name: Add repository to cluster
+  kubernetes.core.helm_repository:
+    name: "{{ item.name }}"
+    repo_url: "{{ item.repo_url }}"
+  loop: "{{ helm.repos }}"
+  become: true
+
+- name: Separately update the repository cache
+  kubernetes.core.helm:
+    kubeconfig_path: "/etc/rancher/k3s/k3s.yaml"
+    name: dummy
+    namespace: kube-system
+    state: absent
+    update_repo_cache: true
+  become: true

ansible-5/roles/prod.k3s/tasks/main.yml

@@ -11,9 +11,99 @@
     opts: "direct-io-mode=disable,_netdev,x-systemd.automount 0 0"
     state: "{{item.state}}"
   with_items: "{{fstab.gluster}}"
+  become: true
 
 # provision docker image registry
+- include_tasks: add_repos.yml
+
+# https://artifacthub.io/packages/helm/twuni/docker-registry
+- name: Deploy latest version of docker-registry in dev-tools namespace
+  local_action:
+    module: kubernetes.core.helm
+    name: dkregistry
+    chart_ref: twuni/docker-registry
+    release_namespace: dev-tools
+    create_namespace: True
+    values:
+      replicaCount: 1
+      ingress:
+        enabled: true
+        hosts:
+          - dkregistry.xai-corp.net
+        className: traefik
+      secrets.htpassword: me1
+      extraVolumeMounts:
+        - name: registry-data
+          mountPath: /var/lib/registry-data
+        - name: registry-auth
+          mountPath: /auth
+        - name: registry-auth
+          mountPath: /etc/docker/registry/
+        - name: letsencrypt-data
+          mountPath: /certs
+        - name: letsencrypt-config
+          mountPath: /etc/letsencrypt
+      extraVolumes:
+        - name: registry-data
+          hostPath:
+            path: /opt/shared/dkregistry/data
+        - name: registry-auth
+          hostPath:
+            path: /opt/shared/dkregistry/auth
+        - name: registry-config
+          hostPath:
+            path: /opt/shared/dkregistry/config/
+        - name: letsencrypt-data
+          hostPath:
+            path: /opt/shared/letsencrypt-2
+        - name: letsencrypt-config
+          hostPath:
+            path: /opt/shared/letsencrypt-2
+#      extraEnvVars:
+#        - name: REGISTRY_HTTP_SECRET
+#          value: aabuioqlwlcpp2
+#        - name: REGISTRY_HTTP_TLS_CERTIFICATE
+#          value: /certs/live/xai-corp.net/fullchain.pem
+#        - name: REGISTRY_HTTP_TLS_KEY
+#          value: /certs/live/xai-corp.net/privkey.pem
+
+
+# k3s config file
+#- name: Copy k3s config file to /etc/rancher/k3s/config.yaml
+#  ansible.builtin.copy:
+#    src: config.yaml
+#    dest: /etc/rancher/k3s/config.yaml
+#  become: true
 
 # provision gitea
 
 # provision argoCD
+
+# provision graphana
+- name: Copy manifest for graphana
+  ansible.builtin.copy:
+    src: manifests/graphana.helm.yaml
+    dest: /var/lib/rancher/k3s/server/manifests/graphana.helm.yaml
+  become: true
+
+- name: Copy manifest for docker registry
+  ansible.builtin.copy:
+    src: manifests/registry.helm.yaml
+    dest: /var/lib/rancher/k3s/server/manifests/dkregistry.helm.yaml
+  become: true
+
+- name: Copy manifest for mariadb
+  ansible.builtin.copy:
+    src: manifests/mariadb.helm.yaml
+    dest: /var/lib/rancher/k3s/server/manifests/mariadb.helm.yaml
+  become: true
+
+- name: remove manifests
+  ansible.builtin.file:
+    state: absent
+    path: "{{ item }}"
+  loop:
+    - /var/lib/rancher/k3s/server/manifests/graphana.helm.yaml
+    - /var/lib/rancher/k3s/server/manifests/dkregistry.helm.yaml
+    - /var/lib/rancher/k3s/server/manifests/mariadb.helm.yaml
+  become: true