QingCloudAppcenter · hlwanghl · May 19, 2020 · Mar 17, 2020 · May 19, 2020
diff --git a/README.md b/README.md
@@ -1,2 +1,12 @@
-# QingCloud Kubernetes Engine(KubeSphere App)
-[KubeSphere](https://kubesphere.io/) 是 QingCloud 开发的基于 Kubernetes 的开源企业级多租户容器平台。 通过 QingCloud AppCenter 能够为用户快速搭建 KubeSphere 环境。此 App 是基于 Kubernetes v1.13.5 预装了 KubeSphere Advanced Edition v2.0.2。可以通过[使用文档](https://docs.qingcloud.com/product/container/)了解详情。
+<p align="center">
+  <img src="./logo.png" alt="Logo" />
+</p>
+<h1 align="center">KubeSphere®️ (QKE)</h1>
+
+[KubeSphere](https://kubesphere.io/) 是一款面向云原生设计的开源项目，在目前主流容器调度平台 Kubernetes 之上构建的分布式多租户容器管理平台，提供简单易用的操作界面以及向导式操作方式，在降低用户使用容器调度平台学习成本的同时，极大降低开发、测试、运维的日常工作的复杂度。
+
+[KubeSphere®️ (QKE) 服务](https://appcenter.qingcloud.com/apps/app-cmgbd5k2)，基于 KubeSphere 的企业级多租户容器管理平台，用户可基于此服务来进行 Kubernetes 集群运维，容器相关的应用开发、部署、升级、CI/CD 和微服务治理等。QKE 充分整合青云底层的 SDN 和 SDS 能力，并基于 OpenPitrix 应用交付框架和运营管理平台提供了容器应用本身的全生命周期管理能力。  
+
+[点此](https://appcenter.qingcloud.com/apps/app-cmgbd5k2) 立即部署。
+
+[点此](https://docs.qingcloud.com/product/container/qke/) 阅读使用文档了解更多详情。
diff --git a/ansible/.gitignore b/ansible/.gitignore
@@ -0,0 +1,3 @@
+**/files/tmp
+/make.retry
+hosts
diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
@@ -0,0 +1,150 @@
+app_version: 2.0.0
+gcr_mirror: kubesphere
+etcd_version: 3.2.24
+cni_version: 0.7.5
+cri_version: 1.13.0
+k8s_version: 1.16.7
+coredns_version: 1.6.0
+calico_version: 3.7.3
+qingcloud_csi_version: 1.1.1
+qingcloud_ccm_version: 1.4.4
+ks_version: 2.1.1
+helm_version: 2.14.3
+helm_stable_repo: https://charts.kubesphere.io/mirror
+nodelocaldns_version: 1.15.5
+nvidia_tesla_version: 418.116.00
+nvidia_plugin_version: 1.0.0-beta4
+binaries:
+- /opt/crictl
+- /opt/etcd
+- /opt/helm
+- /opt/k8s
+docker_images_k8s:
+- calico/cni:v3.7.3
+- calico/kube-controllers:v3.7.3
+- calico/node:v3.7.3
+- coredns/coredns:1.6.0
+- csiplugin/csi-qingcloud:v1.1.1
+- kubesphere/cloud-controller-manager:v1.4.4
+- kubesphere/hyperkube:v1.16.7
+- kubesphere/k8s-dns-node-cache:1.15.5
+- kubesphere/metrics-server-amd64:v0.3.1
+- kubesphere/nfs-client-provisioner:v3.1.0-k8s1.11
+- kubesphere/nginx-ingress-controller:0.24.1
+- kubesphere/pause:3.1
+- kubesphere/tiller:v2.14.3
+- mirrorgooglecontainers/defaultbackend-amd64:1.4
+- mirrorgooglecontainers/pause-amd64:3.1
+- nvidia/k8s-device-plugin:1.0.0-beta4
+- quay.io/coreos/flannel:v0.11.0-amd64
+- quay.io/coreos/flannel-cni:v0.3.0-amd64
+- quay.io/k8scsi/csi-attacher:v2.0.0
+- quay.io/k8scsi/csi-node-driver-registrar:v1.2.0
+- quay.io/k8scsi/csi-provisioner:v1.4.0
+- quay.io/k8scsi/csi-resizer:v0.2.0
+- quay.io/k8scsi/csi-snapshotter:v1.2.2
+docker_images_ks:
+- alpine:3.10.4
+- busybox:1.31.1
+- dduportal/bats:0.4.0
+- docker:19.03
+- docker.elastic.co/kibana/kibana-oss:6.7.0
+- fluent/fluentd:v1.4.2-2.0
+- grafana/grafana:5.2.4
+- haproxy:2.0.4
+- istio/citadel:1.3.3
+- istio/galley:1.3.3
+- istio/kubectl:1.3.3
+- istio/mixer:1.3.3
+- istio/node-agent-k8s:1.3.3
+- istio/pilot:1.3.3
+- istio/proxy_init:1.3.3
+- istio/proxyv2:1.3.3
+- istio/sidecar_injector:1.3.3
+- jaegertracing/jaeger-agent:1.13
+- jaegertracing/jaeger-collector:1.13
+- jaegertracing/jaeger-operator:1.13.1
+- jaegertracing/jaeger-query:1.13
+- java:openjdk-8-jre-alpine
+- jenkins/jenkins:2.176.2
+- jenkins/jnlp-slave:3.27-1
+- joosthofman/wget:1.0
+- kubesphere/addon-resizer:1.8.4
+- kubesphere/alert_adapter:v2.1.0
+- kubesphere/alerting-dbinit:v2.1.0
+- kubesphere/alerting:v2.1.0
+- kubesphere/builder-base:v2.1.0
+- kubesphere/builder-go:v2.1.0
+- kubesphere/builder-maven:v2.1.0
+- kubesphere/builder-nodejs:v2.1.0
+- kubesphere/configmap-reload:v0.0.1
+- kubesphere/configmap-reload:v0.3.0
+- kubesphere/elasticsearch-curator:v5.7.6
+- kubesphere/elasticsearch-oss:6.7.0-1
+- kubesphere/etcd:v3.2.18
+- kubesphere/examples-bookinfo-details-v1:1.13.0
+- kubesphere/examples-bookinfo-productpage-v1:1.13.0
+- kubesphere/examples-bookinfo-ratings-v1:1.13.0
+- kubesphere/examples-bookinfo-reviews-v1:1.13.0
+- kubesphere/examples-bookinfo-reviews-v2:1.13.0
+- kubesphere/examples-bookinfo-reviews-v3:1.13.0
+- kubesphere/fluentbit-operator:v0.1.0
+- kubesphere/fluent-bit:v1.3.2-reload
+- kubesphere/fluent-bit:v1.3.5-reload
+- kubesphere/java-11-centos7:v2.1.0
+- kubesphere/java-11-runtime:v2.1.0
+- kubesphere/java-8-centos7:v2.1.0
+- kubesphere/java-8-r
10000
untime:v2.1.0
+- kubesphere/jenkins-uc:v2.1.1
+- kubesphere/k8s-prometheus-adapter-amd64:v0.4.1
+- kubesphere/ks-account:v2.1.1
+- kubesphere/ks-apigateway:v2.1.1
+- kubesphere/ks-apiserver:v2.1.1
+- kubesphere/ks-console:v2.1.1
+- kubesphere/ks-controller-manager:v2.1.1
+- kubesphere/ks-devops:flyway-v2.1.0
+- kubesphere/ks-installer:v2.1.1
+- kubesphere/kubectl:v1.0.0
+- kubesphere/kube-rbac-proxy:v0.4.1
+- kubesphere/kube-state-metrics:v1.7.2
+- kubesphere/log-sidecar-injector:1.0
+- kubesphere/netshoot:v1.0
+- kubesphere/node-exporter:ks-v0.16.0
+- kubesphere/nodejs-4-centos7:v2.1.0
+- kubesphere/nodejs-6-centos7:v2.1.0
+- kubesphere/nodejs-8-centos7:v2.1.0
+- kubesphere/notification:flyway_v2.1.0
+- kubesphere/notification:v2.1.0
+- kubesphere/prometheus-config-reloader:v0.34.0
+- kubesphere/prometheus-operator:v0.34.0
+- kubesphere/prometheus:v2.5.0
+- kubesphere/python-27-centos7:v2.1.0
+- kubesphere/python-34-centos7:v2.1.0
+- kubesphere/python-35-centos7:v2.1.0
+- kubesphere/python-36-centos7:v2.1.0
+- kubesphere/s2i-binary:v2.1.0
+- kubesphere/s2ioperator:v2.1.1
+- kubesphere/s2irun:v2.1.1
+- kubesphere/tomcat85-java11-centos7:v2.1.0
+- kubesphere/tomcat85-java11-runtime:v2.1.0
+- kubesphere/tomcat85-java8-centos7:v2.1.0
+- kubesphere/tomcat85-java8-runtime:v2.1.0
+- minio/mc:RELEASE.2019-08-07T23-14-43Z
+- minio/minio:RELEASE.2019-08-07T01-59-21Z
+- mirrorgooglecontainers/addon-resizer:1.8.3
+- mirrorgooglecontainers/cluster-proportional-autoscaler-amd64:1.6.0
+- mirrorgooglecontainers/hpa-example:latest
+- mirrorgooglecontainers/metrics-server-amd64:v0.3.3
+- mysql:8.0.11
+- nginx:1.14-alpine
+- nginxdemos/hello:plain-text
+- openpitrix/openpitrix:flyway-v0.4.8
+- openpitrix/openpitrix:v0.4.8
+- openpitrix/release-app:v0.4.3
+- openpitrix/runtime-provider-kubernetes:v0.1.3
+- osixia/openldap:1.3.0
+- perl:latest
+- postgres:9.6.8
+- redis:5.0.5-alpine
+- sonarqube:7.4-community
+- wordpress:4.8-apache
diff --git a/ansible/make.yml b/ansible/make.yml
@@ -0,0 +1,46 @@
+---
+- name: pull docker images
+  hosts: builder
+  roles:
+  - docker-1.0.5
+  - docker-images
+
+- hosts: k8s-client,k8s-node,gpu-node
+  vars:
+    target_env: "{{ lookup('env', 'target') }}"
+  roles:
+  - update-apt-sources-1.0.0
+  - disable-apt-jobs-1.0.0
+  - disable-motd-1.0.0
+  - app-agent-1.0.6
+  - appctl-1.1.7
+  - arping-1.0.5
+  - jq-1.0.9
+  - yq-1.0.6
+  - helm
+
+- name: k8s masters and workers
+  hosts: k8s-node,gpu-node
+  vars:
+    target_env: "{{ lookup('env', 'target') }}"
+  roles:
+  - qingcloud-cli-1.0.5
+  - docker-1.0.5
+  - etcd-1.0.7
+  - k8s-node
+  - lbcli
+  - app-role-k8s
+
+- name: k8s gpu nodes
+  hosts: gpu-node
+  vars:
+    target_env: "{{ lookup('env', 'target') }}"
+  roles:
+  - gpu-node
+
+- hosts: k8s-client
+  vars:
+    target_env: "{{ lookup('env', 'target') }}"
+  roles:
+  - k8s-client
+  - app-role-client
diff --git a/ansible/requirements.yml b/ansible/requirements.yml
@@ -0,0 +1,16 @@
+- src: https://qingcloudappcenter.github.io/ansible-roles/app-agent-1.0.6.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/appctl-1.1.7.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/arping-1.0.5.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/confd-files-1.0.6.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/create-service-user-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/docker-1.0.5.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/golang-1.0.3.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/etcd-1.0.7.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/install-1.0.5.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/jq-1.0.9.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/yq-1.0.6.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/disable-apt-jobs-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/disable-motd-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/update-apt-sources-1.0.0.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/qingcloud-cli-1.0.5.tar.gz
+- src: https://qingcloudappcenter.github.io/ansible-roles/set-passwd-1.0.0.tar.gz
diff --git a/ansible/roles/app-role-client/files/etc/confd/conf.d/client.sh.toml b/ansible/roles/app-role-client/files/etc/confd/conf.d/client.sh.toml
@@ -0,0 +1,8 @@
+[template]
+src = "client.sh.tmpl"
+dest = "/opt/app/current/bin/tmpl/client.sh"
+mode = "0700"
+keys = [
+  "/",
+]
+reload_cmd = "/opt/app/current/bin/tmpl/client.sh"
diff --git a/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl b/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/01.node.env.tmpl
@@ -0,0 +1,19 @@
+{{- $lbIpFromV1 := join (getvs "/hosts/client/*/token") "" }}
+{{- $upgradingFromV1 := gt (len (getvs "/upgrade-audit/from_app_version" | filter "(appv-e5ni6ltd|appv-53p2pg79)")) 0 }}
+
+flush /opt/app/current/bin/envs/node.env << NODE_ENV_EOF
+DATA_MOUNTS=""
+NODE_CTL=client
+MY_IP={{ getv "/host/ip" }}
+KS_ENABLED={{ getv "/env/install_kubesphere" "true" }}
+IS_JOINING={{ exists (printf "/adding-hosts/client/%s/sid" (getv "/host/instance_id")) }}
+IS_UPGRADING_FROM_V1={{ $upgradingFromV1 }}
+IS_HA_CLUSTER={{ gt (len (lsdir "/hosts/master")) 1 }}
+LB_IP_FROM_V1={{ $lbIpFromV1 }}
+{{- if $upgradingFromV1 }}
+KS_MODULES_COUNT=7
+{{- else }}
+{{- $extraModules := getvs "/env/extra.modules" }}
+KS_MODULES_COUNT={{ add 1 (len (split (join $extraModules "") "," | filter "ks-*")) }}
+{{- end }}
+NODE_ENV_EOF
diff --git a/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/02.hosts.tmpl b/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/02.hosts.tmpl
@@ -0,0 +1,45 @@
+buildNodeName() {
+  if [[ "$1" =~ ^node_ ]]; then local format="03"; fi
+  printf "%s%${format}d" $(echo $1 | sed -r 's/node_(.).*/worker-\1/') $2
+}
+
+allNodes="$(sort -V << ALL_NODES_EOF
+{{- range $nodeRole := lsdir "/hosts" }}
+{{- range $instanceId := lsdir (printf "/hosts/%s" $nodeRole) }}
+all/
+{{- $nodeRole }}/
+{{- $nodeSid := getv (printf "/hosts/%s/%s/sid" $nodeRole $instanceId) }}
+{{- $nodeSid }}/
+{{- $instanceId }}/$(buildNodeName {{ $nodeRole }} {{ $nodeSid }})/
+{{- getv (printf "/hosts/%s/%s/node_id" $nodeRole $instanceId) }}/
+{{- getv (printf "/hosts/%s/%s/ip" $nodeRole $instanceId) }}
+{{- end }}
+{{- end }}
+ALL_NODES_EOF
+)"
+
+hostsFile=/etc/hosts
+rotate $hostsFile.swap
+sed "/^# >> QKE nodes./,/^# << QKE nodes./d" $hostsFile > $hostsFile.swap
+firstMasterIp="$(echo "$allNodes" | grep ^all/master/1/ | cut -d/ -f7)"
+{{- with $lbIpFromV1 }}
+lbIp={{ . }}
+{{- else }}
+lbIp="$(awk -F/ '{print $2}' $APISERVER_LB_FILE | grep . || echo -n)"
+{{- end }}
+
+printHostEntry() {
+  echo $1$'\t'$2
+}
+
+flush >> $hostsFile.swap << HOSTS_FILE
+# >> QKE nodes. WARNING: this is managed by script and please don't touch manually.
+$(printHostEntry ${lbIp:-$firstMasterIp} loadbalancer)
+$(echo "$allNodes" | awk -F/ '{printf("%s\t%s %s%s\n", $7, $4, $5, $2~/^n/ ? " "$2$3 : "")}')
+
+{{- with (getv "/env/host_aliases" "") }}
+{{ replace . "," "\n" -1 }}
+{{- end }}
+# << QKE nodes. WARNING: this is managed by script and please don't touch manually.
+HOSTS_FILE
+cp $hostsFile.swap $hostsFile
diff --git a/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/03.authorized_keys.tmpl b/ansible/roles/app-role-client/files/etc/confd/templates/client.sh/03.authorized_keys.tmpl
@@ -0,0 +1,13 @@
+authKeysFile=/root/.ssh/authorized_keys
+sed "/^# >> QKE nodes./,/^# << QKE nodes./d" $authKeysFile > $authKeysFile.swap
+flush >> $authKeysFile.swap << AUTH_KEYS_FILE
+# >> QKE nodes. WARNING: this is managed by script and please don't touch manually.
+{{ join (getvs "/env/user.ssh.*") "\n" }}
+{{- range $instanceId := ls "/hosts/master" }}
+{{- if eq (getv (printf "/hosts/master/%s/sid" $instanceId)) "1" }}
+{{ getv (printf "/hosts/master/%s/pub_key" $instanceId) }}
+{{- end }}
+{{- end }}
+# << QKE nodes. WARNING: this is managed by script and please don't touch manually.
+AUTH_KEYS_FILE
+cat $authKeysFile.swap > $authKeysFile