xai-corp.net/provisioning
1
0
Fork 0
Code Issues 9 Releases Activity

working on configuring a k3s cluster

Browse Source
This commit is contained in:
richard
2022-08-02 09:42:08 -04:00
parent 31ed336880
commit 3a0f4a84b1
23 changed files with 562 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ansible-5/roles/k0s/defaults/main.yml Normal file
View File

@@ -0,0 +1,10 @@
---
k0s_version: v1.22.4+k0s.1
k0s_binary_dest: /usr/local/bin/k0s
k0s_config_dir: /etc/k0s
k0s_data_dir: /var/lib/k0s
k0s_libexec_dir: /usr/libexec/k0s/
k0s_use_custom_config: false
artifacts_dir: "{{ inventory_dir }}/artifacts"

40
ansible-5/roles/k0s/tasks/dir_config.yml Normal file
View File

@@ -0,0 +1,40 @@
---
- name: Create k0s Directories
become: true
file:
path: "{{ item }}"
state: directory
mode: 0755
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
loop:
- "{{ k0s_config_dir }}"
- "{{ k0s_data_dir }}"
- "{{ k0s_libexec_dir }}"
- name: Write the custom k0s config file
template:
src: k0s.yaml.j2
dest: "{{ k0s_config_dir }}/k0s.yaml"
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0600
when: k0s_use_custom_config
- name: Generate default k0s config file
become: true
block:
- name: Create default k0s config
register: default_k0s_config
command: k0s default-config > {{ k0s_config_dir }}/k0s.yaml
- name: Store default k0f config
copy:
dest: "{{ k0s_config_dir }}/k0s.yaml"
content: "{{ default_k0s_config.stdout }}"
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0600
when: not k0s_use_custom_config

22
ansible-5/roles/k0s/tasks/download.yml Normal file
View File

@@ -0,0 +1,22 @@
---
- name: Download k0s binary k0s-{{ k0s_version }}-amd64
get_url:
url: https://github.com/k0sproject/k0s/releases/download/{{ k0s_version }}/k0s-{{ k0s_version }}-amd64
dest: "{{ k0s_binary_dest }}"
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0755
when: ansible_facts.architecture == "x86_64"
- name: Download k0s binary k0s-{{ k0s_version }}-arm64
get_url:
url: https://github.com/k0sproject/k0s/releases/download/{{ k0s_version }}/k0s-{{ k0s_version }}-arm64
dest: "{{ k0s_binary_dest }}"
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: 0755
when:
- ( ansible_facts.architecture is search("arm") and
ansible_facts.userspace_bits == "64" ) or
ansible_facts.architecture is search("aarch64")

90
ansible-5/roles/k0s/tasks/initial_controller.yml Normal file
View File

@@ -0,0 +1,90 @@
---
- name: Create k0s initial controller service with install command
register: install_initial_controller_cmd
command: k0s install controller --config {{ k0s_config_dir }}/k0s.yaml {{ extra_args | default(omit) }}
changed_when: install_initial_controller_cmd | length > 0
- name: Setup custom environment variables for systemd unit
include_role:
name: env_setup
- name: Enable and check k0s service
systemd:
name: k0scontroller
daemon_reload: yes
state: restarted
enabled: yes
- name: Wait for k8s apiserver
wait_for:
host: localhost
port: 6443
delay: 15
timeout: 180
- name: Create worker join token
register: worker_join_token
command: k0s token create --role worker --config {{ k0s_config_dir }}/k0s.yaml
changed_when: worker_join_token | length > 0
- name: Store worker join token
set_fact:
join_token_worker: "{{ worker_join_token.stdout }}"
cacheable: yes
- name: Add k0s worker token to dummy host
add_host:
name: "worker_token_holder"
token: "{{ worker_join_token.stdout }}"
- name: Print worker token
debug:
msg: "k0s worker join token is: {{ worker_join_token.stdout }}"
- name: Create controller join token
register: controller_join_token
command: k0s token create --role controller --config {{ k0s_config_dir }}/k0s.yaml
changed_when: controller_join_token | length > 0
- name: Store controller join token
set_fact:
join_token_controller: "{{ controller_join_token.stdout }}"
cacheable: yes
- name: Add k0s controller token to dummy host
add_host:
name: "controller_token_holder"
token: "{{ controller_join_token.stdout }}"
- name: Print controller token
debug:
msg: "k0s controller join token is: {{ controller_join_token.stdout }}"
- name: Copy config file to user home directory
copy:
src: "{{ k0s_data_dir }}/pki/admin.conf"
dest: ~{{ ansible_user }}/k0s-kubeconfig.yml
remote_src: yes
owner: "{{ ansible_user }}"
mode: 0644
- name: Set controller IP in kubeconfig
replace:
path: ~{{ ansible_user }}/k0s-kubeconfig.yml
regexp: 'localhost'
replace: "{{ ansible_host }}"
- name: Copy kubeconfig
fetch:
src: "~{{ ansible_user }}/k0s-kubeconfig.yml"
dest: "{{ artifacts_dir }}/k0s-kubeconfig.yml"
flat: yes
validate_checksum: no
become: no
- name: "print kubeconfig command"
debug:
msg: "To use Cluster: export KUBECONFIG={{ artifacts_dir }}/k0s-kubeconfig.yml"

6
ansible-5/roles/k0s/tasks/main.yml Normal file
View File

@@ -0,0 +1,6 @@
---
#main install of k0s
- include_tasks: download.yml
- include_tasks: dir_config.yml

76
ansible-5/roles/k0s/templates/k0s.yaml.j2 Normal file
View File

@@ -0,0 +1,76 @@
apiVersion: k0s.k0sproject.io/v1beta1
kind: ClusterConfig
metadata:
creationTimestamp: null
name: k0s
spec:
api:
address: 192.168.4.15
k0sApiPort: 9443
port: 6443
sans:
- 192.168.4.15
controllerManager: {}
images:
calico:
cni:
image: docker.io/calico/cni
version: v3.18.1
kubecontrollers:
image: docker.io/calico/kube-controllers
version: v3.18.1
node:
image: docker.io/calico/node
version: v3.18.1
coredns:
image: k8s.gcr.io/coredns/coredns
version: v1.7.0
default_pull_policy: IfNotPresent
konnectivity:
image: k8s.gcr.io/kas-network-proxy/proxy-agent
version: v0.0.25
kubeproxy:
image: k8s.gcr.io/kube-proxy
version: v1.22.4
kuberouter:
cni:
image: docker.io/cloudnativelabs/kube-router
version: v1.3.2
cniInstaller:
image: quay.io/k0sproject/cni-node
version: 0.1.0
metricsserver:
image: k8s.gcr.io/metrics-server/metrics-server
version: v0.5.0
installConfig:
users:
etcdUser: etcd
kineUser: kube-apiserver
konnectivityUser: konnectivity-server
kubeAPIserverUser: kube-apiserver
kubeSchedulerUser: kube-scheduler
konnectivity:
adminPort: 8133
agentPort: 8132
network:
calico: null
dualStack: {}
kubeProxy:
mode: iptables
kuberouter:
autoMTU: true
mtu: 0
peerRouterASNs: ""
peerRouterIPs: ""
podCIDR: 10.244.0.0/16
provider: kuberouter
serviceCIDR: 10.96.0.0/12
podSecurityPolicy:
defaultPolicy: 00-k0s-privileged
scheduler: {}
storage:
etcd:
peerAddress: 192.168.4.15
type: etcd
telemetry:
enabled: true

8
ansible-5/roles/k3s/tasks/install.yml
View File

@@ -23,7 +23,7 @@
cmd: sh -s --
stdin: "{{ k3s_installer.content }}"
- name: Setup bash completion
ansible.builtin.shell:
cmd: "kubectl completion bash >/etc/bash_completion.d/kubectl"
creates: /etc/bash_completion.d/kubectl
#- name: Setup bash completion
# ansible.builtin.shell:
# cmd: "kubectl completion bash >/etc/bash_completion.d/kubectl"
# creates: /etc/bash_completion.d/kubectl

34
ansible-5/roles/k3s/tasks/install_helm.yml Normal file
View File

@@ -0,0 +1,34 @@
---
#$ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
#$ chmod 700 get_helm.sh
#$ ./get_helm.sh
- name: Install required packages
apt:
name: "{{ item }}"
update_cache: yes
cache_valid_time: 3600
state: latest
with_items:
- curl
- bash-completion
- name: Fetch helm install script
ansible.builtin.uri:
url: https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
return_content: yes
creates: /usr/local/bin/helm
register: helm_installer
- debug:
var: helm_installer
- name: Run installer
async: 300
poll: 10
ansible.builtin.shell:
cmd: bash -s --
stdin: "{{ helm_installer.content }}"
# creates: /usr/local/bin/helm
when: helm_installer.changed

5
ansible-5/roles/k3s/tasks/main.yml
View File

@@ -7,9 +7,12 @@
register: k3s_service
- include_tasks: install.yml
# when: not k3s_service.stat.exists
when: not k3s_service.stat.exists
- name: Start service k3s, if not started
ansible.builtin.service:
name: k3s
state: started
- include_tasks: install_helm.yml

1
ansible-5/roles/ns.xai-corp.net/templates/xai-corp.net.internal.j2
View File

@@ -23,6 +23,7 @@ garden IN A 192.168.4.20
; bare metal servers
home IN A 192.168.4.11
cubox-i IN A 192.168.4.12
cubox-m IN A 192.168.4.15
; virtual machine servers
home02 IN A 192.168.4.22

20
ansible-5/roles/prod.k3s/defaults/main.yml
View File

@@ -3,13 +3,25 @@
fstab:
gluster:
- name: jenkins
path: "/var/lib/jenkins"
state: mounted
# - name: jenkins
# path: "/var/lib/jenkins"
# state: mounted
- name: gitea
path: "/var/lib/gitea"
state: present
- name: vmshares
path: "/opt/shared glusterfs"
path: "/opt/shared"
state: present
helm:
repos:
- name: twuni
repo_url: https://helm.twun.io
- name: jetstack
repo_url: https://charts.jetstack.io
- name: gitea-charts
repo_url: https://dl.gitea.io/charts/
- name: bitnami
repo_url: https://charts.bitnami.com/bitnami
- name: cetic
repo_url: https://cetic.github.io/helm-charts

7
ansible-5/roles/prod.k3s/files/config.yaml Normal file
View File

@@ -0,0 +1,7 @@
write-kubeconfig-mode: 644
#disable:
# - traefik
#token: "secret"
#node-ip: 10.0.10.22,2a05:d012:c6f:4655:d73c:c825:a184:1b75
#cluster-cidr: 10.42.0.0/16,2001:cafe:42:0::/56
#service-cidr: 10.43.0.0/16,2001:cafe:42:1::/112

19
ansible-5/roles/prod.k3s/files/manifests/graphana.helm.yaml Normal file
View File

@@ -0,0 +1,19 @@
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: grafana
namespace: kube-system
spec:
chart: stable/grafana
targetNamespace: monitoring
set:
adminPassword: "NotVerySafePassword"
valuesContent: |-
image:
tag: master
env:
GF_EXPLORE_ENABLED: true
adminUser: admin
sidecar:
datasources:
enabled: true

26
ansible-5/roles/prod.k3s/files/manifests/mariadb.helm.yaml Normal file
View File

@@ -0,0 +1,26 @@
#https://github.com/bitnami/charts/tree/master/bitnami/mariadb
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: mariadb
namespace: kube-system
spec:
chart: bitnami/mariadb
targetNamespace: persistence
set:
replicaCount: 1
valuesContent: |-
image:
tag: 10.5
auth:
rootPassword: "aifuoqibcqobcqb3"
ingress:
className: traefik
hosts: sql.xai-corp.net
extraVolumeMounts:
- name: mysql
mountPath: /var/lib/mysql
extraVolumes:
- name: mysql
hostPath:
path: /opt/mariadb/data

3
ansible-5/roles/prod.k3s/files/manifests/note.txt Normal file
View File

@@ -0,0 +1,3 @@
We can add things to the cluster by adding charts to the mainifests folder. These could be k8s resource definitions or helm charts
Are components removed if the chart is removed? - no

46
ansible-5/roles/prod.k3s/files/manifests/registry.helm.yaml Normal file
View File

@@ -0,0 +1,46 @@
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: dkregistry
namespace: kube-system
spec:
chart: twuni/docker-registry
targetNamespace: dev-tools
set:
adminPassword: "NotVerySafePassword"
replicaCount: 1
valuesContent: |-
image:
tag: 2.7.1
metrics:
enabled: true
ingress:
className: traefik
hosts: dkregistry.xai-corp.net
extraVolumeMounts:
- name: registry-data
mountPath: /var/lib/registry-data
- name: registry-auth
mountPath: /auth
- name: registry-auth
mountPath: /etc/docker/registry/config.yml
- name: letsencrypt-data
mountPath: /certs
- name: letsencrypt-config
mountPath: /etc/letsencrypt
extraVolumes:
- name: registry-data
hostPath:
path: /opt/shared/dkregistry/data
- name: registry-auth
hostPath:
path: /opt/shared/dkregistry/auth
- name: registry-config
hostPath:
path: /opt/shared/dkregistry/config/config.yml
- name: letsencrypt-data
hostPath:
path: /opt/shared/letsencrypt-2
- name: letsencrypt-config
hostPath:
path: /opt/shared/letsencrypt-2

24
ansible-5/roles/prod.k3s/tasks/add_repos.yml Normal file
View File

@@ -0,0 +1,24 @@
---
- name: Add repository locally
local_action:
module: kubernetes.core.helm_repository
name: "{{ item.name }}"
repo_url: "{{ item.repo_url }}"
loop: "{{ helm.repos }}"
- name: Add repository to cluster
kubernetes.core.helm_repository:
name: "{{ item.name }}"
repo_url: "{{ item.repo_url }}"
loop: "{{ helm.repos }}"
become: true
- name: Separately update the repository cache
kubernetes.core.helm:
kubeconfig_path: "/etc/rancher/k3s/k3s.yaml"
name: dummy
namespace: kube-system
state: absent
update_repo_cache: true
become: true

90
ansible-5/roles/prod.k3s/tasks/main.yml
View File

@@ -11,9 +11,99 @@
opts: "direct-io-mode=disable,_netdev,x-systemd.automount 0 0"
state: "{{item.state}}"
with_items: "{{fstab.gluster}}"
become: true
# provision docker image registry
- include_tasks: add_repos.yml
# https://artifacthub.io/packages/helm/twuni/docker-registry
- name: Deploy latest version of docker-registry in dev-tools namespace
local_action:
module: kubernetes.core.helm
name: dkregistry
chart_ref: twuni/docker-registry
release_namespace: dev-tools
create_namespace: True
values:
replicaCount: 1
ingress:
enabled: true
hosts:
- dkregistry.xai-corp.net
className: traefik
secrets.htpassword: me1
extraVolumeMounts:
- name: registry-data
mountPath: /var/lib/registry-data
- name: registry-auth
mountPath: /auth
- name: registry-auth
mountPath: /etc/docker/registry/
- name: letsencrypt-data
mountPath: /certs
- name: letsencrypt-config
mountPath: /etc/letsencrypt
extraVolumes:
- name: registry-data
hostPath:
path: /opt/shared/dkregistry/data
- name: registry-auth
hostPath:
path: /opt/shared/dkregistry/auth
- name: registry-config
hostPath:
path: /opt/shared/dkregistry/config/
- name: letsencrypt-data
hostPath:
path: /opt/shared/letsencrypt-2
- name: letsencrypt-config
hostPath:
path: /opt/shared/letsencrypt-2
# extraEnvVars:
# - name: REGISTRY_HTTP_SECRET
# value: aabuioqlwlcpp2
# - name: REGISTRY_HTTP_TLS_CERTIFICATE
# value: /certs/live/xai-corp.net/fullchain.pem
# - name: REGISTRY_HTTP_TLS_KEY
# value: /certs/live/xai-corp.net/privkey.pem
# k3s config file
#- name: Copy k3s config file to /etc/rancher/k3s/config.yaml
# ansible.builtin.copy:
# src: config.yaml
# dest: /etc/rancher/k3s/config.yaml
# become: true
# provision gitea
# provision argoCD
# provision graphana
- name: Copy manifest for graphana
ansible.builtin.copy:
src: manifests/graphana.helm.yaml
dest: /var/lib/rancher/k3s/server/manifests/graphana.helm.yaml
become: true
- name: Copy manifest for docker registry
ansible.builtin.copy:
src: manifests/registry.helm.yaml
dest: /var/lib/rancher/k3s/server/manifests/dkregistry.helm.yaml
become: true
- name: Copy manifest for mariadb
ansible.builtin.copy:
src: manifests/mariadb.helm.yaml
dest: /var/lib/rancher/k3s/server/manifests/mariadb.helm.yaml
become: true
- name: remove manifests
ansible.builtin.file:
state: absent
path: "{{ item }}"
loop:
- /var/lib/rancher/k3s/server/manifests/graphana.helm.yaml
- /var/lib/rancher/k3s/server/manifests/dkregistry.helm.yaml
- /var/lib/rancher/k3s/server/manifests/mariadb.helm.yaml
become: true