diff --git a/.gitignore b/.gitignore index 648a292..d2f1235 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,33 @@ +<<<<<<< HEAD +<<<<<<< HEAD +down/* +!down/download.sh +!down/offline_images + +# k8s binaries directory +bin/* +!bin/readme.md + +# ansible hosts +hosts + +# k8s storage manifests +manifests/storage/* +!manifests/storage/test.yaml + +# k8s backup directory +roles/cluster-backup/files/* +!roles/cluster-backup/files/readme.md + +# role based variable settings, exclude roles/os-harden/vars/ +/roles/*/vars/* +!/roles/os-harden/vars/ + +# cluster backups +.cluster/ +======= +======= +>>>>>>> 53dfdbb4e1f8b5c083287ab4fca256fab9cdc731 # built application files *.apk *.ap_ @@ -27,3 +57,7 @@ proguard/ *.ipr *.iws .idea/ +<<<<<<< HEAD +>>>>>>> Initial commit +======= +>>>>>>> 53dfdbb4e1f8b5c083287ab4fca256fab9cdc731 diff --git a/01.prepare.yml b/01.prepare.yml new file mode 100644 index 0000000..fd1bcef --- /dev/null +++ b/01.prepare.yml @@ -0,0 +1,24 @@ +# [optional] to synchronize time of nodes with 'chrony' +- hosts: all + roles: + - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes'" } + +# to create CA, kubeconfig, kube-proxy.kubeconfig etc. on 'deploy' node +- hosts: deploy + roles: + - deploy + +# prepare tasks for all nodes +- hosts: + - kube-master + - kube-node + - deploy + - etcd + - lb + roles: + - prepare + +# [optional] to install loadbalance service, only needed by multi-master cluster +- hosts: lb + roles: + - lb diff --git a/02.etcd.yml b/02.etcd.yml new file mode 100644 index 0000000..34a5941 --- /dev/null +++ b/02.etcd.yml @@ -0,0 +1,4 @@ +# to install etcd cluster +- hosts: etcd + roles: + - etcd diff --git a/03.docker.yml b/03.docker.yml new file mode 100644 index 0000000..02f3a7e --- /dev/null +++ b/03.docker.yml @@ -0,0 +1,6 @@ +# to install docker service +- hosts: + - kube-master + - kube-node + roles: + - docker diff --git a/04.kube-master.yml b/04.kube-master.yml new file mode 100644 index 0000000..232ecfe --- /dev/null +++ b/04.kube-master.yml @@ -0,0 +1,16 @@ +# to set up 'kube-master' nodes +- hosts: kube-master + roles: + - kube-master + - kube-node + tasks: + - name: Making master nodes SchedulingDisabled + shell: "{{ bin_dir }}/kubectl cordon {{ inventory_hostname }} " + delegate_to: "{{ groups.deploy[0] }}" + when: DEPLOY_MODE != "allinone" + ignore_errors: true + + - name: Setting master role name + shell: "{{ bin_dir }}/kubectl label node {{ inventory_hostname }} kubernetes.io/role=master --overwrite" + ignore_errors: true + delegate_to: "{{ groups.deploy[0] }}" diff --git a/05.kube-node.yml b/05.kube-node.yml new file mode 100644 index 0000000..5c7ddd3 --- /dev/null +++ b/05.kube-node.yml @@ -0,0 +1,4 @@ +# to set up 'kube-node' nodes +- hosts: kube-node + roles: + - kube-node diff --git a/06.network.yml b/06.network.yml new file mode 100644 index 0000000..f683c36 --- /dev/null +++ b/06.network.yml @@ -0,0 +1,9 @@ +# to install network plugin, only one can be choosen +- hosts: + - kube-master + - kube-node + roles: + - { role: calico, when: "CLUSTER_NETWORK == 'calico'" } + - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" } + - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" } + - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" } diff --git a/07.cluster-addon.yml b/07.cluster-addon.yml new file mode 100644 index 0000000..efe756f --- /dev/null +++ b/07.cluster-addon.yml @@ -0,0 +1,5 @@ +# to install clust-addons +- hosts: + - kube-node + roles: + - cluster-addon diff --git a/1.txt b/1.txt new file mode 100644 index 0000000..7c4a013 --- /dev/null +++ b/1.txt @@ -0,0 +1 @@ +aaa \ No newline at end of file diff --git a/11.harbor.yml b/11.harbor.yml new file mode 100644 index 0000000..289580c --- /dev/null +++ b/11.harbor.yml @@ -0,0 +1,41 @@ +# [optional] to set up a HARBOR, and to integrate the HARBOR with k8s cluster +# read the guide: 'guide/harbor.md' + +- hosts: harbor + roles: + - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes' and NEW_INSTALL == 'yes'" } + - { role: prepare, when: "NEW_INSTALL == 'yes'" } + - { role: docker, when: "NEW_INSTALL == 'yes'" } + - { role: harbor, when: "NEW_INSTALL == 'yes'" } + tasks: + - name: Fetching the HARBOR SERVER's CA cert + fetch: + src: "{{ ca_dir }}/ca.pem" + dest: "{{ base_dir }}/down/" + flat: yes + +- hosts: + - kube-master + - kube-node + tasks: + - name: Define 'harbor_host', a domain + set_fact: harbor_host="{{ hostvars[groups.harbor[0]]['HARBOR_DOMAIN'] }}" + + - name: Define 'harbor_host', an IP Addr + set_fact: harbor_host="{{ groups['harbor'][0] }}" + when: "hostvars[groups.harbor[0]]['HARBOR_DOMAIN'] == ''" + + - name: Creating cert dir of the HARBOR SERVER for the docker daemon + file: name=/etc/docker/certs.d/{{ harbor_host }} state=directory + + - name: Installing the HARBOR SERVER's cert on k8s nodes + copy: src={{ base_dir }}/down/ca.pem dest=/etc/docker/certs.d/{{ harbor_host }}/ca.crt + + # [optional] if you have a DNS server, add an 'A record' instead + - name: Adding an '/etc/hosts' entry for the HARBOR DOMAIN + lineinfile: + dest: /etc/hosts + state: present + regexp: '{{ harbor_host }}' + line: "{{ groups['harbor'][0] }} {{ harbor_host }}" + when: "hostvars[groups.harbor[0]]['HARBOR_DOMAIN'] != ''" diff --git a/22.upgrade.yml b/22.upgrade.yml new file mode 100644 index 0000000..6a92d3a --- /dev/null +++ b/22.upgrade.yml @@ -0,0 +1,23 @@ +# WARNING: Upgrade the k8s cluster can be risky. Make sure you know what you are doing. +# Read the guide: 'op/upgrade.md' . + +# update kubectl binary +- hosts: + - kube-master + - kube-node + - deploy + roles: + - prepare + +# update masters +- hosts: + - kube-master + roles: + - kube-master + - kube-node + +# update nodes +- hosts: + - kube-node + roles: + - { role: kube-node, when: "DEPLOY_MODE != 'allinone'" } diff --git a/23.backup.yml b/23.backup.yml new file mode 100644 index 0000000..3abad10 --- /dev/null +++ b/23.backup.yml @@ -0,0 +1,51 @@ +# cluster-backup playbook +# read the guide: 'op/cluster_restore.md' + +- hosts: + - etcd + roles: + - cluster-backup + +- hosts: + - deploy + tasks: + - name: Creating backup dirs + file: name={{ item }} state=directory + with_items: + - "{{ base_dir }}/roles/cluster-backup/files/ca" + - "{{ base_dir }}/roles/cluster-backup/files/hosts" + - "{{ base_dir }}/roles/cluster-backup/files/snapshot" + + - name: Backing up CA sth + copy: + src: "{{ ca_dir }}/{{ item }}" + dest: "{{ base_dir }}/roles/cluster-backup/files/ca/{{ item }}" + with_items: + - ca.pem + - ca-key.pem + - ca.csr + - ca-csr.json + - ca-config.json + + - name: Backing up ansible hosts-1 + copy: + src: "{{ base_dir }}/hosts" + dest: "{{ base_dir }}/roles/cluster-backup/files/hosts/hosts" + register: p + + - name: Backing up ansible hosts-2 + shell: "cd {{ base_dir }}/roles/cluster-backup/files/hosts && \ + cp -fp hosts hosts-$(date +'%Y%m%d%H%M')" + when: 'p is changed' + + - name: Backing up etcd snapshot-1 + copy: + src: "{{ base_dir }}/roles/cluster-backup/files/snapshot.db" + dest: "{{ base_dir }}/roles/cluster-backup/files/snapshot/snapshot.db" + register: q + + - name: Backing up etcd snapshot-2 + shell: "cd {{ base_dir }}/roles/cluster-backup/files/ && \ + mv -f snapshot.db snapshot/snapshot-$(date +'%Y%m%d%H%M').db" + when: 'q is changed' + diff --git a/24.restore.yml b/24.restore.yml new file mode 100644 index 0000000..4b0467b --- /dev/null +++ b/24.restore.yml @@ -0,0 +1,74 @@ +# cluster-restore playbook +# read the guide: 'op/cluster_restore.md' + +# to restore CA sth on 'deploy' node +- hosts: deploy + tasks: + - name: Restoring dirs of CA sth + file: name=/etc/kubernetes/ssl/ state=directory + + - name: Restoring CA sth + copy: + src: "{{ base_dir }}/roles/cluster-backup/files/ca/{{ item }}" + dest: "{{ ca_dir }}/{{ item }}" + with_items: + - ca.pem + - ca-key.pem + - ca.csr + - ca-csr.json + - ca-config.json + +- hosts: deploy + roles: + - deploy + +# pre-tasks on all nodes +- hosts: all + roles: + - prepare + +# [optional] only needed by multi-master cluster +- hosts: lb + roles: + - lb + +# to install etcd cluster +- hosts: etcd + roles: + - etcd + +# to install docker +- hosts: + - kube-master + - kube-node + roles: + - docker + +# to set up 'kube-master' nodes +- hosts: + - kube-master + roles: + - kube-master + - kube-node + # + tasks: + - name: Making master nodes SchedulingDisabled + shell: "{{ bin_dir }}/kubectl cordon {{ inventory_hostname }} " + when: DEPLOY_MODE != "allinone" + ignore_errors: true + + - name: Setting master role name + shell: "{{ bin_dir }}/kubectl label node {{ inventory_hostname }} kubernetes.io/role=master --overwrite" + ignore_errors: true + +# to set up 'kube-node' nodes +- hosts: + - kube-node + roles: + - kube-node + +# to restore data of etcd cluster +- hosts: etcd + roles: + - cluster-restore + diff --git a/90.setup.yml b/90.setup.yml new file mode 100644 index 0000000..c9e28cb --- /dev/null +++ b/90.setup.yml @@ -0,0 +1,76 @@ +# [optional] to synchronize time of nodes with 'chrony' +- hosts: all + roles: + - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes'" } + +# to create CA, kubeconfig, kube-proxy.kubeconfig etc. on 'deploy' node +- hosts: deploy + roles: + - deploy + +# prepare tasks for all nodes +- hosts: + - kube-master + - kube-node + - deploy + - etcd + - lb + roles: + - prepare + +# [optional] to install loadbalance service, only needed by multi-master cluster +- hosts: lb + roles: + - lb + +# to install etcd cluster +- hosts: etcd + roles: + - etcd + +# to install docker service +- hosts: + - kube-master + - kube-node + roles: + - docker + +# to set up 'kube-master' nodes +- hosts: kube-master + roles: + - kube-master + - kube-node + # + tasks: + - name: Making master nodes SchedulingDisabled + shell: "{{ bin_dir }}/kubectl cordon {{ inventory_hostname }} " + delegate_to: "{{ groups.deploy[0] }}" + when: DEPLOY_MODE != "allinone" + ignore_errors: true + + - name: Setting master role name + shell: "{{ bin_dir }}/kubectl label node {{ inventory_hostname }} kubernetes.io/role=master --overwrite" + ignore_errors: true + delegate_to: "{{ groups.deploy[0] }}" + +# to set up 'kube-node' nodes +- hosts: kube-node + roles: + - { role: kube-node, when: "DEPLOY_MODE != 'allinone'" } + +# to install network plugin, only one can be choosen +- hosts: + - kube-master + - kube-node + roles: + - { role: calico, when: "CLUSTER_NETWORK == 'calico'" } + - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" } + - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" } + - { role: kube-router, when: "CLUSTER_NETWORK == 'kube-router'" } + +# to install clust-addons +- hosts: + - kube-node + roles: + - cluster-addon + diff --git a/99.clean.yml b/99.clean.yml new file mode 100644 index 0000000..ae10ffc --- /dev/null +++ b/99.clean.yml @@ -0,0 +1,214 @@ +# WARNING: This playbook will erase the entire k8s-cluster, include PODs, ETCD data etc. +# Make sure you know what you are doing. + +# to clean 'kube-node' nodes +- hosts: + - kube-master + - kube-node + tasks: + - name: stop and disable kube-node service + service: name={{ item }} state=stopped enabled=no + with_items: + - kubelet + - kube-proxy + ignore_errors: true + + - name: umount kubelet filesystems + shell: "mount | grep '/var/lib/kubelet'| awk '{print $3}'|xargs umount" + args: + warn: false + ignore_errors: true + + - name: remove files and dirs of 'kube-node' nodes + file: name={{ item }} state=absent + with_items: + - "/var/lib/kubelet/" + - "/var/lib/kube-proxy/" + - "/etc/systemd/system/kubelet.service" + - "/etc/systemd/system/kube-proxy.service" + - "/opt/kube/kube-system/" + +# to clean 'kube-master' nodes +- hosts: + - kube-master + tasks: + - name: stop and disable kube-master service + service: name={{ item }} state=stopped enabled=no + with_items: + - kube-apiserver + - kube-controller-manager + - kube-scheduler + ignore_errors: true + + - name: remove files and dirs of 'kube-master' nodes + file: name={{ item }} state=absent + with_items: + - "/var/run/kubernetes" + - "/etc/systemd/system/kube-apiserver.service" + - "/etc/systemd/system/kube-controller-manager.service" + - "/etc/systemd/system/kube-scheduler.service" + +# to clean docker service and networking +- hosts: + - kube-master + - kube-node + tasks: + - name: clean 'kube-router' stuff + shell: "{{ bin_dir }}/docker run --privileged --net=host cloudnativelabs/kube-router --cleanup-config" + ignore_errors: true + when: "CLUSTER_NETWORK == 'kube-router'" + + - name: 获取是否运行名为'kubeasz'的容器 + shell: 'docker ps|grep kubeasz || echo "NOT FOUND"' + register: install_info + + - name: stop and disable docker service + service: + name: docker + state: stopped + enabled: no + ignore_errors: true + when: "'kubeasz' not in install_info.stdout" + + # as k8s-network-plugins use host-network, '/var/run/docker/netns/default' must be umounted + - name: unmount docker filesystem-1 + mount: path=/var/run/docker/netns/default state=unmounted + when: "'kubeasz' not in install_info.stdout" + + - name: unmount docker filesystem-2 + mount: path=/var/lib/docker/overlay state=unmounted + when: "'kubeasz' not in install_info.stdout" + + - name: remove files and dirs + file: name={{ item }} state=absent + with_items: + - "/var/lib/docker/" + - "/var/run/docker/" + - "/etc/systemd/system/docker.service" + - "/etc/systemd/system/docker.service.requires/" + - "/etc/systemd/system/docker.service.d/" + - "/etc/bash_completion.d/docker" + - "/usr/bin/docker" + when: "'kubeasz' not in install_info.stdout" + + - name: remove files and dirs2 + file: name={{ item }} state=absent + with_items: + - "/etc/cni/" + - "/run/flannel/" + - "/etc/calico/" + - "/var/run/calico/" + - "/var/lib/calico/" + - "/var/log/calico/" + - "/etc/cilium/" + - "/var/run/cilium/" + - "/sys/fs/bpf/tc/" + - "/var/lib/cni/" + - "/var/lib/kube-router/" + - "/opt/kube/kube-system/" + + - name: cleanup iptables + shell: "iptables -F && iptables -X \ + && iptables -F -t nat && iptables -X -t nat \ + && iptables -F -t raw && iptables -X -t raw \ + && iptables -F -t mangle && iptables -X -t mangle" + when: "'kubeasz' not in install_info.stdout" + + - name: cleanup networks1 + shell: "ip link del tunl0; \ + ip link del flannel.1; \ + ip link del cni0; \ + ip link del mynet0; \ + ip link del kube-bridge; \ + ip link del dummy0; \ + ip link del kube-ipvs0; \ + ip link del cilium_net; \ + ip link del cilium_vxlan" + ignore_errors: true + + - name: cleanup networks2 + shell: "ip link del docker0; \ + systemctl restart networking; \ + systemctl restart network" + ignore_errors: true + when: "'kubeasz' not in install_info.stdout" + + - name: cleanup 'calico' routes + shell: "for rt in `ip route|grep bird|sed 's/blackhole//'|awk '{print $1}'`;do ip route del $rt;done;" + when: "CLUSTER_NETWORK == 'calico'" + ignore_errors: true + +# to clean 'etcd' nodes +- hosts: etcd + tasks: + - name: stop and disable etcd service + service: + name: etcd + state: stopped + enabled: no + ignore_errors: true + + - name: remove files and dirs + file: name={{ item }} state=absent + with_items: + - "/var/lib/etcd" + - "/etc/etcd/" + - "/backup/k8s" + - "/etc/systemd/system/etcd.service" + +# to clean 'lb' nodes +- hosts: + - lb + - ex-lb + tasks: + - name: stop keepalived service + shell: systemctl disable keepalived && systemctl stop keepalived + ignore_errors: true + + - name: stop haproxy service + shell: systemctl disable haproxy && systemctl stop haproxy + ignore_errors: true + + - name: remove files and dirs + file: name={{ item }} state=absent + with_items: + - "/etc/haproxy" + - "/etc/keepalived" + +# to clean ntp, certs and keys, env path +- hosts: + - kube-master + - kube-node + - deploy + - etcd + - lb + tasks: + - name: stop and disable chrony in Ubuntu + service: name=chrony state=stopped enabled=no + ignore_errors: true + tags: rm_ntp + when: ansible_distribution == "Ubuntu" or ansible_distribution == "Debian" + + - name: stop and disable chronyd in CentOS/RedHat + service: name=chronyd state=stopped enabled=no + ignore_errors: true + tags: rm_ntp + when: ansible_distribution == "CentOS" or ansible_distribution == "RedHat" + + - name: clean certs and keys + file: name={{ item }} state=absent + with_items: + - "/etc/kubernetes/" + - "{{ ca_dir }}" + - "/root/.kube/config" + - "/etc/docker/" + + - name: clean 'ENV PATH' + lineinfile: + dest: ~/.bashrc + state: absent + regexp: '{{ item }}' + with_items: + - 'kubeasz' + - 'helm' + - 'kubectl completion' diff --git a/README.md b/README.md index 7473b15..54e0c10 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,116 @@ +<<<<<<< HEAD +<<<<<<< HEAD +# kubeasz - Ansible Spirit Zone In Kubernetes + +`kubeasz`致力于提供快速部署高可用`k8s`集群的工具, 并且也努力成为`k8s`实践、使用的参考书；基于二进制方式部署和利用`ansible-playbook`实现自动化：即提供一键安装脚本, 也可以分步执行安装各个组件, 同时讲解每一步主要参数配置和注意事项。 + +**集群特性：`TLS`双向认证、`RBAC`授权、多`Master`高可用、支持`Network Policy`、备份恢复** + +|组件|支持| +|:-|:-| +|OS|Ubuntu 16.04+, CentOS/RedHat 7| +|k8s|v1.8, v1.9, v1.10, v1.11, v1.12, v1.13| +|etcd|v3.1, v3.2, v3.3| +|docker|17.03.x-ce, 18.06.x-ce, 18.09.x| +|network|calico, cilium, flannel, kube-router| + +- 注：集群用到的所有二进制文件已打包好供下载 [https://pan.baidu.com/s/1c4RFaA](https://pan.baidu.com/s/1c4RFaA) + +## 快速指南 + +单机快速体验k8s集群的测试、开发环境--[AllinOne部署](docs/setup/quickStart.md) + +## 安装指南 + + + + + + + + + + + + + + +
00-规划集群和安装概览02-安装etcd集群04-安装master节点06-安装集群网络
01-创建证书和安装准备03-安装docker服务05-安装node节点07-安装集群插件
+ +- 公有云上部署请阅读 [使用kubeasz在公有云上创建k8s集群](docs/setup/kubeasz_on_public_cloud.md) +- 容器方式部署请阅读 [使用kubeasz容器创建k8s集群](docs/setup/docker_kubeasz.md) + +## 使用指南 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
常用插件+DNSdashboardmetrics-serverprometheusefkingress
集群管理+增加node节点增加master节点管理etcd集群删除节点升级集群备份恢复
特性实验NetworkPolicyRollingUpdateHPA
周边生态harborhelmjenkinsgitlab
应用实践java应用部署elasticsearch集群mariadb集群
+ +## 沟通交流 + +- 微信群：k8s&kubeasz实践, 搜索微信号`badtobone`, 请备注（城市-github用户名）, 验证通过会加入群聊。 +- 推荐阅读：[feisky-Kubernetes指南](https://github.com/feiskyer/kubernetes-handbook/blob/master/SUMMARY.md) [rootsongjc-Kubernetes指南](https://github.com/rootsongjc/kubernetes-handbook) [opsnull-安装教程](https://github.com/opsnull/follow-me-install-kubernetes-cluster) + +## 贡献&致谢 + +请阅读[项目分支说明](docs/mixes/branch.md), 欢迎提[Issues](https://github.com/gjmzj/kubeasz/issues)和[PRs](docs/mixes/HowToContribute.md)参与维护项目！感谢您的关注与支持！ + +- [如何 PR](docs/mixes/HowToContribute.md) +- [如何捐赠](docs/mixes/donate.md) + +Copyright 2017 gjmzj (jmgaozz@163.com) Apache License 2.0, 详情见 [LICENSE](docs/mixes/LICENSE) 文件。 +======= +======= +>>>>>>> 53dfdbb4e1f8b5c083287ab4fca256fab9cdc731 VAB === //ZHOU FENG VoiceAddreBook +<<<<<<< HEAD +>>>>>>> Initial commit +======= +>>>>>>> 53dfdbb4e1f8b5c083287ab4fca256fab9cdc731 diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..af38cc3 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,264 @@ +# config file for ansible -- http://ansible.com/ +# ============================================== + +# nearly all parameters can be overridden in ansible-playbook +# or with command line flags. ansible will read ANSIBLE_CONFIG, +# ansible.cfg in the current working directory, .ansible.cfg in +# the home directory or /etc/ansible/ansible.cfg, whichever it +# finds first + +[defaults] + +# some basic default values... + +#inventory = /etc/ansible/hosts +#library = /usr/share/my_modules/ +#remote_tmp = $HOME/.ansible/tmp +#forks = 5 +#poll_interval = 15 +#sudo_user = root +#ask_sudo_pass = True +#ask_pass = True +#transport = smart +#remote_port = 22 +#module_lang = C + +# plays will gather facts by default, which contain information about +# the remote system. +# +# smart - gather by default, but don't regather if already gathered +# implicit - gather by default, turn off with gather_facts: False +# explicit - do not gather by default, must say gather_facts: True +gathering = smart + +# additional paths to search for roles in, colon separated +roles_path = /etc/ansible/roles + +# uncomment this to disable SSH key host checking +host_key_checking = False + +# change the default callback +#stdout_callback = skippy +# enable additional callbacks +#callback_whitelist = timer, mail + +# change this for alternative sudo implementations +#sudo_exe = sudo + +# What flags to pass to sudo +# WARNING: leaving out the defaults might create unexpected behaviours +#sudo_flags = -H -S -n + +# SSH timeout +#timeout = 10 + +# default user to use for playbooks if user is not specified +# (/usr/bin/ansible will use current user as default) +#remote_user = root + +# logging is off by default unless this path is defined +# if so defined, consider logrotate +#log_path = /var/log/ansible.log + +# default module name for /usr/bin/ansible +#module_name = command + +# use this shell for commands executed under sudo +# you may need to change this to bin/bash in rare instances +# if sudo is constrained +#executable = /bin/sh + +# if inventory variables overlap, does the higher precedence one win +# or are hash values merged together? The default is 'replace' but +# this can also be set to 'merge'. +#hash_behaviour = replace + +# by default, variables from roles will be visible in the global variable +# scope. To prevent this, the following option can be enabled, and only +# tasks and handlers within the role will see the variables there +#private_role_vars = yes + +# list any Jinja2 extensions to enable here: +#jinja2_extensions = jinja2.ext.do,jinja2.ext.i18n + +# if set, always use this private key file for authentication, same as +# if passing --private-key to ansible or ansible-playbook +#private_key_file = /path/to/file + +# format of string {{ ansible_managed }} available within Jinja2 +# templates indicates to users editing templates files will be replaced. +# replacing {file}, {host} and {uid} and strftime codes with proper values. +#ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host} +# This short version is better used in templates as it won't flag the file as changed every run. +#ansible_managed = Ansible managed: {file} on {host} + +# by default, ansible-playbook will display "Skipping [host]" if it determines a task +# should not be run on a host. Set this to "False" if you don't want to see these "Skipping" +# messages. NOTE: the task header will still be shown regardless of whether or not the +# task is skipped. +display_skipped_hosts = False + +# by default (as of 1.3), Ansible will raise errors when attempting to dereference +# Jinja2 variables that are not set in templates or action lines. Uncomment this line +# to revert the behavior to pre-1.3. +#error_on_undefined_vars = False + +# by default (as of 1.6), Ansible may display warnings based on the configuration of the +# system running ansible itself. This may include warnings about 3rd party packages or +# other conditions that should be resolved if possible. +# to disable these warnings, set the following value to False: +#system_warnings = True + +# by default (as of 1.4), Ansible may display deprecation warnings for language +# features that should no longer be used and will be removed in future versions. +# to disable these warnings, set the following value to False: +#deprecation_warnings = True + +# (as of 1.8), Ansible can optionally warn when usage of the shell and +# command module appear to be simplified by using a default Ansible module +# instead. These warnings can be silenced by adjusting the following +# setting or adding warn=yes or warn=no to the end of the command line +# parameter string. This will for example suggest using the git module +# instead of shelling out to the git command. +# command_warnings = False + + +# set plugin path directories here, separate with colons +#action_plugins = /usr/share/ansible/plugins/action +#callback_plugins = /usr/share/ansible/plugins/callback +#connection_plugins = /usr/share/ansible/plugins/connection +#lookup_plugins = /usr/share/ansible/plugins/lookup +#vars_plugins = /usr/share/ansible/plugins/vars +#filter_plugins = /usr/share/ansible/plugins/filter +#test_plugins = /usr/share/ansible/plugins/test + +# by default callbacks are not loaded for /bin/ansible, enable this if you +# want, for example, a notification or logging callback to also apply to +# /bin/ansible runs +#bin_ansible_callbacks = False + + +# don't like cows? that's unfortunate. +# set to 1 if you don't want cowsay support or export ANSIBLE_NOCOWS=1 +#nocows = 1 + +# set which cowsay stencil you'd like to use by default. When set to 'random', +# a random stencil will be selected for each task. The selection will be filtered +# against the `cow_whitelist` option below. +#cow_selection = default +#cow_selection = random + +# when using the 'random' option for cowsay, stencils will be restricted to this list. +# it should be formatted as a comma-separated list with no spaces between names. +# NOTE: line continuations here are for formatting purposes only, as the INI parser +# in python does not support them. +#cow_whitelist=bud-frogs,bunny,cheese,daemon,default,dragon,elephant-in-snake,elephant,eyes,\ +# hellokitty,kitty,luke-koala,meow,milk,moofasa,moose,ren,sheep,small,stegosaurus,\ +# stimpy,supermilker,three-eyes,turkey,turtle,tux,udder,vader-koala,vader,www + +# don't like colors either? +# set to 1 if you don't want colors, or export ANSIBLE_NOCOLOR=1 +#nocolor = 1 + +# if set to a persistent type (not 'memory', for example 'redis') fact values +# from previous runs in Ansible will be stored. This may be useful when +# wanting to use, for example, IP information from one group of servers +# without having to talk to them in the same playbook run to get their +# current IP information. +#fact_caching = memory + + +# retry files +# When a playbook fails by default a .retry file will be created in ~/ +# You can disable this feature by setting retry_files_enabled to False +# and you can change the location of the files by setting retry_files_save_path + +retry_files_enabled = False +#retry_files_save_path = ~/.ansible-retry + + +# prevents logging of task data, off by default +#no_log = False + +# prevents logging of tasks, but only on the targets, data is still logged on the master/controller +#no_target_syslog = False + +# controls the compression level of variables sent to +# worker processes. At the default of 0, no compression +# is used. This value must be an integer from 0 to 9. +#var_compression_level = 9 + +[privilege_escalation] +#become=True +#become_method=sudo +#become_user=root +#become_ask_pass=False + +[paramiko_connection] + +# uncomment this line to cause the paramiko connection plugin to not record new host +# keys encountered. Increases performance on new host additions. Setting works independently of the +# host key checking setting above. +#record_host_keys=False + +# by default, Ansible requests a pseudo-terminal for commands executed under sudo. Uncomment this +# line to disable this behaviour. +#pty=False + +[ssh_connection] + +# ssh arguments to use +# Leaving off ControlPersist will result in poor performance, so use +# paramiko on older platforms rather than removing it +ssh_args = -o ControlMaster=auto -o ControlPersist=3600s + +# The path to use for the ControlPath sockets. This defaults to +# "%(directory)s/ansible-ssh-%%h-%%p-%%r", however on some systems with +# very long hostnames or very long path names (caused by long user names or +# deeply nested home directories) this can exceed the character limit on +# file socket names (108 characters for most platforms). In that case, you +# may wish to shorten the string below. +# +# Example: +# control_path = %(directory)s/%%h-%%r +control_path = /tmp/ansible-ssh-%%h-%%p-%%r + +# Enabling pipelining reduces the number of SSH operations required to +# execute a module on the remote server. This can result in a significant +# performance improvement when enabled, however when using "sudo:" you must +# first disable 'requiretty' in /etc/sudoers +# +# By default, this option is disabled to preserve compatibility with +# sudoers configurations that have requiretty (the default on many distros). +# +pipelining = True + +# if True, make ansible use scp if the connection type is ssh +# (default is sftp) +#scp_if_ssh = True + +# if False, sftp will not use batch mode to transfer files. This may cause some +# types of file transfer failures impossible to catch however, and should +# only be disabled if your sftp version has problems with batch mode +sftp_batch_mode = True + +[accelerate] +#accelerate_port = 5099 +#accelerate_timeout = 30 +#accelerate_connect_timeout = 5.0 + +# The daemon timeout is measured in minutes. This time is measured +# from the last activity to the accelerate daemon. +#accelerate_daemon_timeout = 30 + +# If set to yes, accelerate_multi_key will allow multiple +# private keys to be uploaded to it, though each user must +# have access to the system via SSH to add a new key. The default +# is "no". +#accelerate_multi_key = yes + +[selinux] +# file systems that require special treatment when dealing with security context +# the default behaviour that copies the existing context or uses the user default +# needs to be changed to use the file system dependent context. +#special_context_filesystems=nfs,vboxsf,fuse,ramfs diff --git a/bin/readme.md b/bin/readme.md new file mode 100755 index 0000000..0cd1fcc --- /dev/null +++ b/bin/readme.md @@ -0,0 +1,3 @@ +# Binaries for installing k8s + +Binaries can be downloaded at https://pan.baidu.com/s/1c4RFaA, alternatively they can be downloaded from official github repos separately. diff --git a/dockerfiles/ansible/Dockerfile b/dockerfiles/ansible/Dockerfile new file mode 100644 index 0000000..fbf468e --- /dev/null +++ b/dockerfiles/ansible/Dockerfile @@ -0,0 +1,29 @@ +# Dockerfile for building Ansible image for Alpine 3 +# Origin from https://github.com/William-Yeh/docker-ansible + +FROM alpine:3.8 + +RUN \ + echo "===> Adding Python runtime..." && \ + apk --update add python py-pip openssl ca-certificates && \ + apk --update add --virtual build-dependencies \ + python-dev libffi-dev openssl-dev build-base && \ + pip install --upgrade pip cffi && \ + \ + \ + echo "===> Installing Ansible..." && \ + pip install ansible==2.6.12 && \ + \ + \ + echo "===> Installing handy tools..." && \ + pip install --upgrade pycrypto && \ + apk --update add bash openssh-client rsync && \ + \ + \ + echo "===> Removing package list..." && \ + apk del build-dependencies && \ + rm -rf /var/cache/apk/* && \ + rm -rf /root/.cache + +# default command: display Ansible version +CMD [ "ansible", "--version" ] diff --git a/dockerfiles/kubeasz/Dockerfile b/dockerfiles/kubeasz/Dockerfile new file mode 100644 index 0000000..f3c62dc --- /dev/null +++ b/dockerfiles/kubeasz/Dockerfile @@ -0,0 +1,12 @@ +# NOTE: Prepare following Requirements and Build the image "kubeasz:$TAG" +# Requirement 1: git clone https://github.com/gjmzj/kubeasz.git +# Requirement 2: download binaries at https://pan.baidu.com/s/1c4RFaA, and put into dir 'kubeasz/bin' +# Build: docker build -t kubeasz:$TAG . + +FROM jmgao1983/ansible:v2.6 + +COPY kubeasz/ /etc/ansible + +RUN ln -s /etc/ansible/tools/easzctl /usr/bin/easzctl + +CMD [ "sleep", "360000000" ] diff --git a/docs/guide/chrony.md b/docs/guide/chrony.md new file mode 100644 index 0000000..684cdc2 --- /dev/null +++ b/docs/guide/chrony.md @@ -0,0 +1,146 @@ +# chrony 时间同步 + +在安装k8s集群前需确保各节点时间同步；`chrony` 是一个优秀的 `NTP` 实现，性能比ntp好，且配置管理方便；它既可作时间服务器服务端，也可作客户端。 + +- `OpenStack` 社区也推荐使用 `chrony`实现各节点之间的时间同步 + +## 安装配置介绍 + +项目中选定一个节点(`deploy` )作为集群内部其他节点的时间同步源，而 deploy节点本身从公网源同步；当然如果整个集群都无法访问公网，那么请手动校准deploy 节点的时间后，仍旧可以作为内部集群的时间源服务器。 + +- 配置 chrony server, 在`/etc/chrony.conf` 配置以下几项，其他项默认值即可 + +``` bash +# 1. 配置时间源，国内可以增加阿里的时间源 ntp1.aliyun.com +server {{ ntp_server }} iburst + +# 2. 配置允许同步的客户端网段 +allow {{ local_network }} + +# 3. 配置离线也能作为源服务器 +local stratum 10 +``` + +- 配置 chrony client + +``` bash +# 1. 清除所有其他时间源，只配置一个本地 deploy节点作为源 +server {{ groups.deploy[0] }} iburst + +# 2. 其他所有项可以默认配置 +``` + +## `kubeasz` 集成安装 + +- 修改 ansible hosts 文件，在 `deploy` 节点配置 `NTP_ENABLED=yes` (默认: no) +- [可选] 修改 roles/chrony/var/main.yml 中的变量定义，关于文件 roles/chrony/var/main.yml 的由来请看[这里](../setup/config_guide.md) + +对于新集群或者新节点，`chrony` 的安装配置已经集成到 `90.setup.yml` `01.prepare.yml` `20.addnode.yml` `21.addmaster.yml` 等脚本中；对于已运行中的集群请执行如下命令进行安装： + +`ansible-playbook /etc/ansible/roles/chrony/chrony.yml ` + +## 验证配置 + +- 在 chrony server (deploy 节点)检查时间源信息，默认配置为`ntp1.aliyun.com`的地址： + +``` +$ chronyc sources -v +210 Number of sources = 5 + + .-- Source mode '^' = server, '=' = peer, '#' = local clock. + / .- Source state '*' = current synced, '+' = combined , '-' = not combined, +| / '?' = unreachable, 'x' = time may be in error, '~' = time too variable. +|| .- xxxx [ yyyy ] +/- zzzz +|| Reachability register (octal) -. | xxxx = adjusted offset, +|| Log2(Polling interval) --. | | yyyy = measured offset, +|| \ | | zzzz = estimated error. +|| | | \ +MS Name/IP address Stratum Poll Reach LastRx Last sample +=============================================================================== +^* 120.25.115.20 2 9 377 55 +147us[ +250us] +/- 15ms +^- 85.199.214.100 1 10 377 182 -25ms[ -24ms] +/- 128ms +^- makaki.miuku.net 2 10 367 307 +61ms[ +61ms] +/- 127ms +^- static-5-103-139-163.ip.f 1 9 167 572 +532us[ +336us] +/- 117ms +^- 119.28.183.184 2 7 377 33 -130us[ -130us] +/- 47ms +``` + +- 在 chrony server (deploy 节点)检查时间源同步状态 + +``` +chronyc sourcestats -v +210 Number of sources = 5 + .- Number of sample points in measurement set. + / .- Number of residual runs with same sign. + | / .- Length of measurement set (time). + | | / .- Est. clock freq error (ppm). + | | | / .- Est. error in freq. + | | | | / .- Est. offset. + | | | | | | On the -. + | | | | | | samples. \ + | | | | | | | +Name/IP Address NP NR Span Frequency Freq Skew Offset Std Dev +============================================================================== +120.25.115.20 15 11 44m +0.011 0.909 +4097ns 758us +85.199.214.100 22 13 49m -3.588 5.097 -23ms 5709us +makaki.miuku.net 22 14 46m +2.455 6.225 +64ms 4945us +static-5-103-139-163.ip.f 20 13 42m -2.472 10.168 +3615us 6732us +119.28.183.184 16 9 19m +10.378 25.190 +3469us 6803us +``` + +- 在 chrony client 检查，可以看到时间源只有一个（deploy 节点地址） + +``` +$ chronyc sources +210 Number of sources = 1 +MS Name/IP address Stratum Poll Reach LastRx Last sample +=============================================================================== +^* 192.168.1.1 3 6 377 15 +4085ns[ -25us] +/- 15ms +$ chronyc sourcestats +210 Number of sources = 1 +Name/IP Address NP NR Span Frequency Freq Skew Offset Std Dev +============================================================================== +192.168.1.1 5 4 323 -0.252 0.819 -3031ns 15us +``` + +## 验证时间同步状态完成 + +chrony 服务启动后，chrony server 会与配置的公网参考时间服务器进行同步；server 同步完成后，chrony client 会与 server 进行时间同步；一般来说整个集群达到时间同步需要几十分钟。可以用如下命令检查，初始时 **NTP synchronized: no**，同步完成后 **NTP synchronized: yes** + +``` bash +$ ansible all -m shell -a 'timedatectl' +192.168.1.1 | SUCCESS | rc=0 >> + Local time: Sat 2019-01-26 11:51:51 HKT + Universal time: Sat 2019-01-26 03:51:51 UTC + RTC time: Sat 2019-01-26 03:51:52 + Time zone: Asia/Hong_Kong (HKT, +0800) + Network time on: yes +NTP synchronized: yes + RTC in local TZ: no + +192.168.1.4 | SUCCESS | rc=0 >> + Local time: Sat 2019-01-26 11:51:51 HKT + Universal time: Sat 2019-01-26 03:51:51 UTC + RTC time: Sat 2019-01-26 03:51:52 + Time zone: Asia/Hong_Kong (HKT, +0800) + Network time on: yes +NTP synchronized: yes + RTC in local TZ: no + +192.168.1.2 | SUCCESS | rc=0 >> + Local time: Sat 2019-01-26 11:51:51 HKT + Universal time: Sat 2019-01-26 03:51:51 UTC + RTC time: Sat 2019-01-26 03:51:52 + Time zone: Asia/Hong_Kong (HKT, +0800) + Network time on: yes +NTP synchronized: yes + RTC in local TZ: no + +192.168.1.3 | SUCCESS | rc=0 >> + Local time: Sat 2019-01-26 11:51:51 HKT + Universal time: Sat 2019-01-26 03:51:51 UTC + RTC time: Sat 2019-01-26 03:51:52 + Time zone: Asia/Hong_Kong (HKT, +0800) + Network time on: yes +NTP synchronized: yes + RTC in local TZ: no +``` diff --git a/docs/guide/dashboard.1.6.3.md b/docs/guide/dashboard.1.6.3.md new file mode 100644 index 0000000..9a29479 --- /dev/null +++ b/docs/guide/dashboard.1.6.3.md @@ -0,0 +1,129 @@ +## dashboard + +本文档基于 dashboard 1.6.3版本，从 1.7.x 版本以后，dashboard 默认开启自带的登陆验证界面，登陆流程差异详见[新版本](dashboard.md)。 + ++ 注意：实际测试k8s版本<=1.9.1支持dashboard 1.6.3, 建议k8s 1.9 以后使用 dashboard 新版本。 + +### 部署 + +``` bash +# 部署dashboard 主yaml配置文件 +$ kubectl create -f /etc/ansible/manifests/dashboard/1.6.3/kubernetes-dashboard.yaml +# 部署基本密码认证配置[可选]，密码文件位于 /etc/kubernetes/ssl/basic-auth.csv +$ kubectl create -f /etc/ansible/manifests/dashboard/ui-admin-rbac.yaml +$ kubectl create -f /etc/ansible/manifests/dashboard/ui-read-rbac.yaml +``` + +请在另外窗口打开 [kubernetes-dashboard.yaml](../../manifests/dashboard/1.6.3/kubernetes-dashboard.yaml) + ++ 由于 kube-apiserver 启用了 RBAC授权，dashboard使用的 ServiceAccount `kubernetes-dashboard` 必须有相应的权限去访问apiserver(在新版本1.8.0中，该访问权限已按最小化方式授权)，在1.6.3 版本，先粗放一点，把`kubernetes-dashboard` 与 集群角色 `cluster-admin` 绑定，这样dashboard就拥有了所有访问apiserver的权限。 ++ 开发测试环境为了方便配置dashboard-service时候，指定 `NodePort`方式暴露服务，这样集群外部可以使用 `http://NodeIP:NodePort` 方式直接访问 dashboard，生产环境建议关闭该访问途径。 + +### 验证 + +``` bash +# 查看pod 运行状态 +kubectl get pod -n kube-system | grep dashboard +kubernetes-dashboard-86bd8778bf-w4974 1/1 Running 0 12h +# 查看dashboard service +kubectl get svc -n kube-system|grep dashboard +kubernetes-dashboard NodePort 10.68.7.67 80:5452/TCP 12h +# 查看集群服务 +kubectl cluster-info|grep dashboard +kubernetes-dashboard is running at https://192.168.1.10:6443/api/v1/namespaces/kube-system/services/kubernetes-dashboard/proxy +# 查看pod 运行日志，关注有没有错误 +kubectl logs kubernetes-dashboard-86bd8778bf-w4974 -n kube-system +``` + +### 访问 + +因为dashboard 作为k8s 原生UI，能够展示各种资源信息，甚至可以有修改、增加、删除权限，所以有必要对访问进行认证和控制，本项目预置部署的集群有以下安全设置：详见 [apiserver配置模板](../../roles/kube-master/templates/kube-apiserver.service.j2) + ++ 启用 `TLS认证` `RBAC授权`等安全特性 ++ 关闭 apiserver非安全端口8080的外部访问`--insecure-bind-address=127.0.0.1` ++ 关闭匿名认证`--anonymous-auth=false` ++ 补充启用基本密码认证 `--basic-auth-file=/etc/kubernetes/ssl/basic-auth.csv`，[密码文件模板](../../roles/kube-master/templates/basic-auth.csv.j2)中按照每行(密码,用户名,序号)的格式，可以定义多个用户 + +#### 1. 临时访问：使用 `http://NodeIP:NodePort` 方式直接访问 dashboard，生产环境建议关闭该途径 + +#### 2. 用户+密码访问：安全性比证书方式差点，务必保管好密码文件`basic-auth.csv` + +- 这里演示两种权限，使用admin 登陆dashboard拥有所有权限，使用readonly 登陆后仅查看权限，首先在 master节点文件 `/etc/kubernetes/ssl/basic-auth.csv` 确认用户名和密码，如果要增加或者修改用户，修改保存该文件后记得逐个重启你的master 节点 +- 为了演示用户密码访问，如果你已经完成证书访问方式，你可以在浏览器删除证书，或者访问时候浏览器询问你证书时不选证书 +- 2.1 设置用户admin 的RBAC 权限，如下运行配置文件 `kubectl create -f ui-admin-rbac.yaml` + +``` bash +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ui-admin +rules: +- apiGroups: + - "" + resources: + - services + - services/proxy + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ui-admin-binding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ui-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: admin +``` +- 2.2 设置用户readonly 的RBAC 权限，如下运行配置文件 `kubectl create -f ui-read-rbac.yaml` + +``` bash +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ui-read +rules: +- apiGroups: + - "" + resources: + - services + - services/proxy + verbs: + - get + - list + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ui-read-binding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ui-read +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: readonly +``` +- 2.3 访问 `https://x.x.x.x:6443/api/v1/namespaces/kube-system/services/kubernetes-dashboard/proxy` 使用 admin登陆拥有所有权限，比如删除某个部署；使用 readonly登陆只有查看权限，尝试删除某个部署会提示错误 `forbidden: User \"readonly\" cannot delete services/proxy in the namespace \"kube-system\"` + +#### 3. 证书访问：最安全的方式，配置较复杂 +- 使用集群CA 生成客户端证书，可以根据需要生成权限不同的证书，这里为了演示直接使用 kubectl使用的证书和key(在03.kubectl.yml阶段生成)，该证书拥有所有权限 +- 指定格式导出该证书，进入`/etc/kubernetes/ssl`目录，使用命令`openssl pkcs12 -export -in admin.pem -inkey admin-key.pem -out kube-admin.p12` 提示输入证书密码和确认密码，可以用密码再增加一层保护，也可以直接回车跳过，完成后目录下多了 `kube-admin.p12`文件，将它分发给授权的用户 +- 用户将 `kube-admin.p12` 双击导入证书即可，`IE` 和`Chrome` 中输入`https://x.x.x.x:6443/api/v1/namespaces/kube-system/services/kubernetes-dashboard/proxy` 或者 `https://x.x.x.x:6443/ui` 即可访问。补充：最新firefox需要在浏览器中单独导入 [选项] - [隐私与安全] - [证书/查看证书] - [您的证书] 页面点击 [导入] 该证书 + +### 小结 + ++ dashboard 版本 1.6.3 访问控制实现较复杂，文档中给出的例子也有助于你理解 RBAC的灵活控制能力，当然最好去[官方文档](https://kubernetes.io/docs/admin/authorization/rbac/)学习一下，这块篇幅不长 ++ 由于还未部署 Heapster 插件，当前 dashboard 不能展示 Pod、Nodes 的 CPU、内存等 metric 图形，后续部署 heapster后自然能够看到 ++ 本文中的权限设置仅供演示用，生产环境请在此基础上修改成适合你安全需求的方式 + diff --git a/docs/guide/dashboard.md b/docs/guide/dashboard.md new file mode 100644 index 0000000..86cc65f --- /dev/null +++ b/docs/guide/dashboard.md @@ -0,0 +1,121 @@ +## dashboard + +本文档基于 dashboard 1.10.0版本，k8s版本 1.11.x。因 dashboard 1.7 以后默认开启了自带的登陆验证机制，因此不同版本登陆有差异： + +- 旧版（<= 1.6）建议通过apiserver访问，直接通过apiserver 认证授权机制去控制 dashboard权限，详见[旧版文档](dashboard.1.6.3.md) +- 新版（>= 1.7）可以使用自带的登陆界面，使用不同Service Account Tokens 去控制访问 dashboard的权限 + +### 部署 + +如果之前已按照本项目部署dashboard1.6.3，先删除旧版本：`kubectl delete -f /etc/ansible/manifests/dashboard/1.6.3/` + +新版配置文件参考[官方文档](https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml) + ++ 增加了通过`api-server`方式访问dashboard ++ 增加了`NodePort`方式暴露服务，这样集群外部可以使用 `https://NodeIP:NodePort` (注意是https不是http，区别于1.6.3版本) 直接访问 dashboard。 + +安装部署 + +``` bash +# 部署dashboard 主yaml配置文件 +$ kubectl apply -f /etc/ansible/manifests/dashboard/kubernetes-dashboard.yaml +# [可选]部署基本密码认证配置，使用apiserver 方式访问需要 +$ kubectl apply -f /etc/ansible/manifests/dashboard/ui-admin-rbac.yaml +$ kubectl apply -f /etc/ansible/manifests/dashboard/ui-read-rbac.yaml +# 创建可读可写 admin Service Account +$ kubectl apply -f /etc/ansible/manifests/dashboard/admin-user-sa-rbac.yaml +# 创建只读 read Service Account +$ kubectl apply -f /etc/ansible/manifests/dashboard/read-user-sa-rbac.yaml +``` + +### 验证 + +``` bash +# 查看pod 运行状态 +kubectl get pod -n kube-system | grep dashboard +kubernetes-dashboard-7c74685c48-9qdpn 1/1 Running 0 22s +# 查看dashboard service +kubectl get svc -n kube-system|grep dashboard +kubernetes-dashboard NodePort 10.68.219.38 443:24108/TCP 53s +# 查看集群服务 +kubectl cluster-info|grep dashboard +kubernetes-dashboard is running at https://192.168.1.1:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy +# 查看pod 运行日志 +kubectl logs kubernetes-dashboard-7c74685c48-9qdpn -n kube-system +``` + ++ 由于还未部署 Heapster 插件，当前 dashboard 不能展示 Pod、Nodes 的 CPU、内存等 metric 图形，后续部署 heapster后自然能够看到 + +### 访问控制 + +因为dashboard 作为k8s 原生UI，能够展示各种资源信息，甚至可以有修改、增加、删除权限，所以有必要对访问进行认证和控制，本项目部署的集群有以下安全设置：详见 [apiserver配置模板](../../roles/kube-master/templates/kube-apiserver.service.j2) + ++ 启用 `TLS认证` `RBAC授权`等安全特性 ++ 关闭 apiserver非安全端口8080的外部访问`--insecure-bind-address=127.0.0.1` ++ 关闭匿名认证`--anonymous-auth=false` ++ 补充启用基本密码认证 `--basic-auth-file=/etc/kubernetes/ssl/basic-auth.csv`，[密码文件模板](../../roles/kube-master/templates/basic-auth.csv.j2)中按照每行(密码,用户名,序号)的格式，可以定义多个用户 + +新版 dashboard可以有多层访问控制，首先与旧版一样可以使用apiserver 方式登陆控制： + ++ 第一步通过api-server本身安全认证流程，与之前[1.6.3版本](dashboard.1.6.3.md)相同，这里不再赘述 ++ 第二步通过dashboard自带的登陆流程，使用`Kubeconfig` `Token`等方式登陆 + +**注意：** 如果集群已启用 ingress tls的话，可以[配置ingress规则访问dashboard](ingress-tls.md#%E9%85%8D%E7%BD%AE-dashboard-ingress) + +### 演示新登陆方式 + +为演示方便这里使用 `https://NodeIP:NodePort` 方式访问 dashboard，支持两种登录方式：Kubeconfig、令牌(Token) + +- 令牌登录（admin） + +选择“令牌(Token)”方式登陆，复制下面输出的admin token 字段到输入框 + +``` bash +# 创建Service Account 和 ClusterRoleBinding +$ kubectl apply -f /etc/ansible/manifests/dashboard/admin-user-sa-rbac.yaml +# 获取 Bearer Token，找到输出中 ‘token:’ 开头那一行 +$ kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') +``` + +- 令牌登录（只读） + +选择“令牌(Token)”方式登陆，复制下面输出的read token 字段到输入框 + +``` bash +# 创建Service Account 和 ClusterRoleBinding +$ kubectl apply -f /etc/ansible/manifests/dashboard/read-user-sa-rbac.yaml +# 获取 Bearer Token，找到输出中 ‘token:’ 开头那一行 +$ kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep read-user | awk '{print $1}') +``` +- Kubeconfig登录（admin） +Admin kubeconfig文件默认位置：`/root/.kube/config`，该文件中默认没有token字段，使用Kubeconfig方式登录，还需要将token追加到该文件中，完整的文件格式如下： +``` +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdxxxxxxxxxxxxxx + server: https://192.168.1.2:6443 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: admin + name: kubernetes +current-context: kubernetes +kind: Config +preferences: {} +users: +- name: admin + user: + client-certificate-data: LS0tLS1CRUdJTiBDRxxxxxxxxxxx + client-key-data: LS0tLS1CRUdJTxxxxxxxxxxxxxx + token: eyJhbGcixxxxxxxxxxxxxxxx +``` + +- Kubeconfig登陆（只读） +首先[创建只读权限 kubeconfig文件](../op/readonly_kubectl.md)，然后类似追加只读token到该文件，略。 + +### 参考 + +- 1. [Dashboard Access control](https://github.com/kubernetes/dashboard/wiki/Access-control) +- 2. [a-read-only-kubernetes-dashboard](https://blog.cowger.us/2018/07/03/a-read-only-kubernetes-dashboard.html) diff --git a/docs/guide/efk.md b/docs/guide/efk.md new file mode 100644 index 0000000..aaa4325 --- /dev/null +++ b/docs/guide/efk.md @@ -0,0 +1,195 @@ +### 第一部分：EFK + +`EFK` 插件是`k8s`项目的一个日志解决方案，它包括三个组件：[Elasticsearch](), [Fluentd](), [Kibana]()；Elasticsearch 是日志存储和日志搜索引擎，Fluentd 负责把`k8s`集群的日志发送给 Elasticsearch, Kibana 则是可视化界面查看和检索存储在 Elasticsearch 的数据。 + +### 准备 + +下载官方最新[release](https://github.com/kubernetes/kubernetes/release)，进入目录: `kubernetes/cluster/addons/fluentd-elasticsearch`，参考官方配置的基础上使用本项目`manifests/efk/`部署，以下为几点主要的修改： + ++ 官方提供的`kibana-deployment.yaml`中的参数`SERVER_BASEPATH`在k8s v1.8 版本以后部署需要按照本项目调整 ++ 修改官方docker镜像，方便国内下载加速 + +### 安装 + +``` bash +$ kubectl create -f /etc/ansible/manifests/efk/ +$ kubectl create -f /etc/ansible/manifests/efk/es-without-pv/ +``` + +**注意**：Fluentd 是以 DaemonSet 形式运行且只会调度到有`beta.kubernetes.io/fluentd-ds-ready=true`标签的节点，所以对需要收集日志的节点逐个打上标签： + +``` bash +$ kubectl label nodes 192.168.1.2 beta.kubernetes.io/fluentd-ds-ready=true +node "192.168.1.2" labeled +``` + +### 验证 + +``` bash +kubectl get pods -n kube-system|grep -E 'elasticsearch|fluentd|kibana' +elasticsearch-logging-0 1/1 Running 0 19h +elasticsearch-logging-1 1/1 Running 0 19h +fluentd-es-v2.0.2-6c95c 1/1 Running 0 17h +fluentd-es-v2.0.2-f2xh8 1/1 Running 0 8h +fluentd-es-v2.0.2-pv5q5 1/1 Running 0 8h +kibana-logging-d5cffd7c6-9lz2p 1/1 Running 0 1m +``` +kibana Pod 第一次启动时会用较长时间(10-20分钟)来优化和 Cache 状态页面，可以查看 Pod 的日志观察进度，如下等待 `Ready` 状态 + +``` bash +$ kubectl logs -n kube-system kibana-logging-d5cffd7c6-9lz2p -f +... +{"type":"log","@timestamp":"2018-03-13T07:33:00Z","tags":["listening","info"],"pid":1,"message":"Server running at http://0:5601"} +{"type":"log","@timestamp":"2018-03-13T07:33:00Z","tags":["status","ui settings","info"],"pid":1,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"} +``` + +### 访问 Kibana + +这里介绍 `kube-apiserver`方式访问，获取访问 URL + +``` bash +$ kubectl cluster-info | grep Kibana +Kibana is running at https://192.168.1.10:8443/api/v1/namespaces/kube-system/services/kibana-logging/proxy +``` +浏览器访问 URL：`https://192.168.1.10:8443/api/v1/namespaces/kube-system/services/kibana-logging/proxy`，然后使用`basic auth（参照hosts文件设置，默认：用户admin 密码test1234）`或者`证书` 的方式认证后即可，关于认证可以参考[dashboard文档](dashboard.md) + +首次登陆需要在`Management` - `Index Patterns` 创建 `index pattern`，可以使用默认的 logstash-* pattern，点击 Create; 创建Index后，稍等几分钟就可以在 Discover 菜单看到 ElasticSearch logging 中汇聚的日志； + +### 第二部分：日志持久化之静态PV +日志数据是存放于 `Elasticsearch POD`中，但是默认情况下它使用的是`emptyDir`存储类型，所以当 `POD`被删除或重新调度时，日志数据也就丢失了。以下讲解使用`NFS` 服务器手动（静态）创建`PV` 持久化保存日志数据的例子。 + +#### 配置 NFS + ++ 准备一个nfs服务器，如果没有可以参考[nfs-server](nfs-server.md)创建。 ++ 配置nfs服务器的共享目录，即修改`/etc/exports`（根据实际网段替换`192.168.1.*`），修改后重启`systemctl restart nfs-server`。 + +``` bash +/share 192.168.1.*(rw,sync,insecure,no_subtree_check,no_root_squash) +/share/es0 192.168.1.*(rw,sync,insecure,no_subtree_check,no_root_squash) +/share/es1 192.168.1.*(rw,sync,insecure,no_subtree_check,no_root_squash) +/share/es2 192.168.1.*(rw,sync,insecure,no_subtree_check,no_root_squash) +``` + +#### 使用静态 PV安装 EFK + +- 请按实际日志容量需求修改 `es-static-pv/es-statefulset.yaml` 文件中 volumeClaimTemplates 设置的 storage: 4Gi 大小 +- 请根据实际nfs服务器地址、共享目录、容量大小修改 `es-static-pv/es-pv*.yaml` 文件中对应的设置 + +``` bash +# 如果之前已经安装了默认的EFK，请用以下两个命令先删除它 +$ kubectl delete -f /etc/ansible/manifests/efk/ +$ kubectl delete -f /etc/ansible/manifests/efk/es-without-pv/ + +# 安装静态PV 的 EFK +$ kubectl create -f /etc/ansible/manifests/efk/ +$ kubectl create -f /etc/ansible/manifests/efk/es-static-pv/ +``` ++ 目录`es-static-pv` 下首先是利用 NFS服务预定义了三个 PV资源，然后在 `es-statefulset.yaml`定义中使用 `volumeClaimTemplates` 去匹配使用预定义的 PV资源；注意 PV参数：`accessModes` `storageClassName` `storage`容量大小必须两边匹配。 + +#### 验证安装 + ++ 1.集群中查看 `pod` `pv` `pvc` 等资源 + +``` bash +$ kubectl get pods -n kube-system|grep -E 'elasticsearch|fluentd|kibana' +elasticsearch-logging-0 1/1 Running 0 10m +elasticsearch-logging-1 1/1 Running 0 10m +fluentd-es-v2.0.2-6c95c 1/1 Running 0 10m +fluentd-es-v2.0.2-f2xh8 1/1 Running 0 10m +fluentd-es-v2.0.2-pv5q5 1/1 Running 0 10m +kibana-logging-d5cffd7c6-9lz2p 1/1 Running 0 10m + +$ kubectl get pv +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pv-es-0 4Gi RWX Recycle Bound kube-system/elasticsearch-logging-elasticsearch-logging-0 es-storage-class 1m +pv-es-1 4Gi RWX Recycle Bound kube-system/elasticsearch-logging-elasticsearch-logging-1 es-storage-class 1m +pv-es-2 4Gi RWX Recycle Available es-storage-class 1m + +$ kubectl get pvc --all-namespaces +NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +kube-system elasticsearch-logging-elasticsearch-logging-0 Bound pv-es-0 4Gi RWX es-storage-class 2m +kube-system elasticsearch-logging-elasticsearch-logging-1 Bound pv-es-1 4Gi RWX es-storage-class 1m +``` + ++ 2.网页访问 `kibana`查看具体的日志，如上须等待（约15分钟） `kibana Pod`优化和 Cache 状态页面，达到 `Ready` 状态。 + ++ 3.登陆 NFS Server 查看对应目录和内部数据 + +``` bash +$ ls /share +es0 es1 es2 +``` + +### 第三部分：日志持久化之动态PV +`PV` 作为集群的存储资源，`StatefulSet` 依靠它实现 POD的状态数据持久化，但是当 `StatefulSet`动态伸缩时，它的 `PVC`请求也会变化，如果每次都需要管理员手动去创建对应的 `PV`资源，那就很不方便；因此 K8S还提供了 `provisioner`来动态创建 `PV`，不仅节省了管理员的时间，还可以根据不同的 `StorageClasses`封装不同类型的存储供 PVC 选用。 + ++ 此功能需要 `API-SERVER` 参数 `--admission-control`字符串设置中包含 `DefaultStorageClass`，本项目中已经开启。 ++ `provisioner`指定 Volume 插件的类型，包括内置插件（如 kubernetes.io/glusterfs）和外部插件（如 external-storage 提供的 ceph.com/cephfs，nfs-client等），以下讲解使用 `nfs-client-provisioner`来动态创建 `PV`来持久化保存 `EFK`的日志数据。 + +#### 配置 NFS（同上） + +确保 `/etc/exports` 配置如下共享目录，并确保 `/share`目录可读可写权限，否则可能因为权限问题无法动态生成 PV的对应目录。（根据实际情况替换IP段`192.168.1.*`） +``` bash +/share 192.168.1.*(rw,sync,insecure,no_subtree_check,no_root_squash) +``` + +#### 使用动态 PV安装 EFK + +- 首先根据[集群存储](../setup/08-cluster-storage.md)创建nfs-client-provisioner +- 然后按实际需求修改 `es-dynamic-pv/es-statefulset.yaml` 文件中 volumeClaimTemplates 设置的 storage: 4Gi 大小 + +``` bash +# 如果之前已经安装了默认的EFK或者静态PV EFK，请用以下命令先删除它 +$ kubectl delete -f /etc/ansible/manifests/efk/ +$ kubectl delete -f /etc/ansible/manifests/efk/es-without-pv/ +$ kubectl delete -f /etc/ansible/manifests/efk/es-static-pv/ + +# 安装动态PV 的 EFK +$ kubectl create -f /etc/ansible/manifests/efk/ +$ kubectl create -f /etc/ansible/manifests/efk/es-dynamic-pv/ +``` ++ 首先 `nfs-client-provisioner.yaml` 创建一个工作 POD，它监听集群的 PVC请求，并当 PVC请求来到时调用 `nfs-client` 去请求 `nfs-server`的存储资源，成功后即动态生成对应的 PV资源。 ++ `nfs-dynamic-storageclass.yaml` 定义 NFS存储类型的类型名 `nfs-dynamic-class`，然后在 `es-statefulset.yaml`中必须使用这个类型名才能动态请求到资源。 + +#### 验证安装 + ++ 1.集群中查看 `pod` `pv` `pvc` 等资源 + +``` bash +$ kubectl get pods -n kube-system|grep -E 'elasticsearch|fluentd|kibana' +elasticsearch-logging-0 1/1 Running 0 10m +elasticsearch-logging-1 1/1 Running 0 10m +fluentd-es-v2.0.2-6c95c 1/1 Running 0 10m +fluentd-es-v2.0.2-f2xh8 1/1 Running 0 10m +fluentd-es-v2.0.2-pv5q5 1/1 Running 0 10m +kibana-logging-d5cffd7c6-9lz2p 1/1 Running 0 10m + +$ kubectl get pv +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-50644f36-358b-11e8-9edd-525400cecc16 4Gi RWX Delete Bound kube-system/elasticsearch-logging-elasticsearch-logging-0 nfs-dynamic-class 10m +pvc-5b105ee6-358b-11e8-9edd-525400cecc16 4Gi RWX Delete Bound kube-system/elasticsearch-logging-elasticsearch-logging-1 nfs-dynamic-class 10m + +$ kubectl get pvc --all-namespaces +NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +kube-system elasticsearch-logging-elasticsearch-logging-0 Bound pvc-50644f36-358b-11e8-9edd-525400cecc16 4Gi RWX nfs-dynamic-class 10m +kube-system elasticsearch-logging-elasticsearch-logging-1 Bound pvc-5b105ee6-358b-11e8-9edd-525400cecc16 4Gi RWX nfs-dynamic-class 10m +``` + ++ 2.网页访问 `kibana`查看具体的日志，如上须等待（约15分钟） `kibana Pod`优化和 Cache 状态页面，达到 `Ready` 状态。 + ++ 3.登陆 NFS Server 查看对应目录和内部数据 + +``` bash +$ ls /share # 可以看到类似如下的目录生成 +kube-system-elasticsearch-logging-elasticsearch-logging-0-pvc-50644f36-358b-11e8-9edd-525400cecc16 +kube-system-elasticsearch-logging-elasticsearch-logging-1-pvc-5b105ee6-358b-11e8-9edd-525400cecc16 +``` + +### 参考 + +1. [EFK 配置](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/fluentd-elasticsearch) +1. [nfs-client-provisioner](https://github.com/kubernetes-incubator/external-storage/tree/master/nfs-client) +1. [persistent-volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) +1. [storage-classes](https://kubernetes.io/docs/concepts/storage/storage-classes/) + + diff --git a/docs/guide/gitlab/app.yaml.md b/docs/guide/gitlab/app.yaml.md new file mode 100644 index 0000000..f6f2225 --- /dev/null +++ b/docs/guide/gitlab/app.yaml.md @@ -0,0 +1,90 @@ +## 3.3 K8S 应用部署模板 app.yaml + +以下示例配置仅做参考，描述一个简单 java spring boot项目的 k8s 部署文件模板；在实际部署前，CI/CD流程中会对变量做替换。详见 [gitlab-ci.yml文件](gitlab-ci.yml.md)。 + +``` bash +cat > .ci/app.yaml << EOF +--- +apiVersion: v1 +kind: Namespace +metadata: + name: PROJECT_NS +--- +apiVersion: v1 +kind: Secret +metadata: + name: harborkey1 + namespace: PROJECT_NS +data: + #待替换的变量DOCKER_KEY，参考 docs/guide/harbor.md#k8s%E4%B8%AD%E4%BD%BF%E7%94%A8harbor + .dockerconfigjson: DOCKER_KEY +type: kubernetes.io/dockerconfigjson + +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: APP_NAME + namespace: PROJECT_NS +spec: + replicas: APP_REP + template: + metadata: + labels: + run: APP_NAME + spec: + containers: + - name: APP_NAME + image: ProjectImage + imagePullPolicy: Always + env: + # 设置java的时区 + - name: TZ + value: "Asia/Shanghai" + resources: + limits: + cpu: 500m + memory: 1600Mi + requests: + cpu: 200m + memory: 800Mi + ports: + - containerPort: 8080 + imagePullSecrets: + - name: harborkey1 + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: APP_NAME + name: APP_NAME + namespace: PROJECT_NS +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: + run: APP_NAME + sessionAffinity: None + +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: APP_NAME-ingress + namespace: PROJECT_NS +spec: + rules: + - host: AppDomain + http: + paths: + - path: /AppPath + backend: + serviceName: APP_NAME + servicePort: 80 +EOF +``` + diff --git a/docs/guide/gitlab/config.sh.md b/docs/guide/gitlab/config.sh.md new file mode 100644 index 0000000..256dd30 --- /dev/null +++ b/docs/guide/gitlab/config.sh.md @@ -0,0 +1,49 @@ +## 3.2 环境配置替换 config.sh + +首先应用开发人员需要整理在不同环境（测试环境/生产环境）的配置参数，并在源代码中约定好替换的名称（如db_host, db_usr）；然后用户必须在项目gitlab web界面（“Settings”>"CI/CD">"Variables"）配置变量；最后根据gitlab-ci.yml文件定义CI/CD执行的需要，编写如下简单变量替换shell脚本；该shell脚本分别在测试环境打包阶段（beta-build）和生产环境打包阶段（prod-build）阶段运行。 + +以下脚本仅作示例，实际应根据项目需要增加/修改需替换变量名称与对应源代码中的配置文件 + +``` bash +cat > .ci/config.sh << EOF +#!/bin/bash + +#set -o verbose +#set -o xtrace + +beta_config() { +sed -i \ + -e "s/db_host/$BETA_DB_HOST/g" \ + -e "s/db_usr/$BETA_DB_USR/g" \ + -e "s/db_pwd/$BETA_DB_PWD/g" \ + example-web/src/main/resources/config/datasource.properties # 项目源码的配置文件 +sed -i \ + -e "s/redis_host/$BETA_REDIS_HOST/g" \ + -e "s/redis_port/$BETA_REDIS_PORT/g" \ + -e "s/redis_pwd/$BETA_REDIS_PWD/g" \ + example-web/src/main/resources/config/redis.properties # 项目源码的配置文件 +} + +prod_config() { +sed -i \ + -e "s/db_host/$PROD_DB_HOST/g" \ + -e "s/db_usr/$PROD_DB_USR/g" \ + -e "s/db_pwd/$PROD_DB_PWD/g" \ + example-web/src/main/resources/config/datasource.properties +sed -i \ + -e "s/redis_host/$PROD_REDIS_HOST/g" \ + -e "s/redis_port/$PROD_REDIS_PORT/g" \ + -e "s/redis_pwd/$PROD_REDIS_PWD/g" \ + example-web/src/main/resources/config/redis.properties +} + +if [[ "$CI_JOB_STAGE" == "beta-build" ]];then + beta_config +elif [[ "$CI_JOB_STAGE" == "prod-build" ]];then + prod_config +else + echo "error: undefined CI_JOB_STAGE!" +fi +EOF +``` + diff --git a/docs/guide/gitlab/gitlab-ci.yml.md b/docs/guide/gitlab/gitlab-ci.yml.md new file mode 100644 index 0000000..cdbe840 --- /dev/null +++ b/docs/guide/gitlab/gitlab-ci.yml.md @@ -0,0 +1,194 @@ +## 3.1 配置 gitlab-ci.yml + +示例应用搭建 CI/CD 流水线的背景需求 + +- 应用测试环境部署在本地k8s平台，生产环境部署在阿里云上k8s平台 +- 应用的多个feature分支可以并行测试 +- 对于即将发布的release分支，本地提供封版测试环境，阿里云上提供UAT测试环境 + +以下示例配置为个人经验总结，仅供参考，可以根据自己的理解和项目需要不断优化完善；总体来说 gitlab-ci.yml 配置很丰富，基本上能够满足各种个性化的CI/CD流程需要。 + +``` bash +$ cat > .ci/gitlab-ci.yml << EOF +variables: ### 定义全局变量 http://gitlab.test.com/help/ci/variables/README.md + PROJECT_NS: '$CI_PROJECT_NAMESPACE-$CI_JOB_STAGE' # 定义项目命名空间，对应k8s的namespace + APP_NAME: '$CI_PROJECT_NAME-$CI_COMMIT_REF_SLUG' # 使用项目名和git提交信息作为应用名 + IMAGE_NAME: '$CI_PROJECT_NAMESPACE-$CI_PROJECT_NAME:$CI_PIPELINE_ID' # 定义镜像名称 + +stages: ### 定义ci各阶段 + - beta-build # beta环境编译打包 + - beta-deploy # beta环境部署 + - beta-feature-delete # beta环境feature分支手动删除 + - prod-build # prod环境编译打包 + - prod-uat-deploy # prod-uat环境部署 + - prod-deploy # prod环境部署 + - prod-rollback # prod回滚 + +job_beta_build: + stage: beta-build # beta环境编译打包 + tags: + - build-shell # 定义带`build-shell`标签的runner可以运行该job + only: # 定义只在如下分支或者tag运行该job + - master + - develop + - /^feature.*$/ + - release + #when: manual # 调试阶段可以先手动，后续可以注释掉以自动运行 + script: ### runner上运行的脚本 + - bash .ci/config.sh # 不同环境配置替换，后文详解 config.sh + - mvn clean install -Dmaven.test.skip=true -U # mvn 编译，可以去runner 虚机上手动执行编译测试 + - mv example-web/target/*.jar dockerfiles/ # 把mvn生成的xxx.jar移动到dockerfiles目录下 + - export IMAGE=`echo $IMAGE_NAME | sed 's/\//-/g'` # 转换镜像名，例：mygroup/java/example:172 >> mygroup-java-example:172 + - cd dockerfiles && docker build -t $BETA_HARBOR/example/$IMAGE . # 创建 docker 镜像 + - docker login -u $BETA_HARBOR_USR -p $BETA_HARBOR_PWD $BETA_HARBOR # 登陆到内部镜像仓库 harbor，并推送 + - docker push $BETA_HARBOR/example/$IMAGE + - docker logout $BETA_HARBOR + +job_push_beta: ### 推送到beta环境，可以推送不同分支 develop, feature-1, ...> + stage: beta-deploy # 可以做到多分支同时测试，甚至最后的release分支也要在beta封版测试 + tags: + - beta-shell # 定义带`beta-shell`标签的runner可以运行该job + only: + - master + - develop + - /^feature.*$/ + - release + when: manual # 调试阶段可以先手动，后续可以注释掉以自动运行 + variables: + BETA_EXP_Domain: '$CI_COMMIT_REF_SLUG.example.test.com' # job内部变量，指定该应用在beta环境的 ingress 域名 + script: + - export IMAGE=`echo $IMAGE_NAME | sed 's/\//-/g'` # 转换 $IMAGE_NAME 中可能的 / 字符 + - export PROJECT_NS=`echo $PROJECT_NS | sed 's/\//-/g'` # 转换命名空间中可能有的 / 字符 + # 替换beta环境的参数配置 + - sed -i "s/PROJECT_NS/$PROJECT_NS/g" .ci/app.yaml ### app.yaml 即k8s的部署模板文件，详见后面 app.yaml.md 文档，注意这里的变量有的来自> + - sed -i "s/APP_NAME/$APP_NAME/g" .ci/app.yaml # gitlab 系统变量, 有的是在项目 CI/CD 设置里面用户定义的变量 + - sed -i "s/APP_REP/$BETA_APP_REP/g" .ci/app.yaml + - sed -i "s/AppDomain/$BETA_EXP_Domain/g" .ci/app.yaml + - sed -i "s/ProjectImage/$BETA_HARBOR\/example\/$IMAGE/g" .ci/app.yaml + - sed -i "s/DOCKER_KEY/$BETA_KEY/g" .ci/app.yaml # DOCKER_KEY 为k8s平台能从镜像仓库pull所需的认证信息，详见harbor文档 + # + - mkdir -p /opt/kube/$PROJECT_NS/$APP_NAME # 在runner：beta-shell虚机本地创建应用配置目录，调试检查用 + - cp -f .ci/app.yaml /opt/kube/$PROJECT_NS/$APP_NAME + - kubectl --kubeconfig=/etc/.beta/config apply -f .ci/app.yaml # 部署应用（runner虚机上预先配置了kubectl权限执行测试k8s平台） + +job_delete_beta: ### 多测试环境并行部署在beta k8s平台，feature分支测试完毕后删除代码分支， + stage: beta-feature-delete # 同时需要删除该分支在k8s平台上的部署，可以由开发人员自行执行该job删除 + tags: + - beta-shell + only: + - /^feature.*$/ + when: manual + script: + - export PROJECT_NS=`echo $PROJECT_NS | sed 's/\//-/g'` + - kubectl --kubeconfig=/etc/.beta/config delete deploy,svc,ing $APP_NAME -n $PROJECT_NS + +job_prod_build: ### prod环境编译打包，这里prod环境我们使用阿里云上的K8S + stage: prod-build # 阿里云k8s平台上运行的uat环境和正式环境都使用本次打包镜像 + tags: + - build-shell + only: # 仅master和release分支可以执行该job + - master + - release + #when: manual + script: + - bash .ci/config.sh # config.sh 会执行替换生产环境的变量 + - mvn clean install -Dmaven.test.skip=true -U # mvn 编译，可以去runner 虚机上手动执行编译测试 + - mv example-web/target/*.jar dockerfiles/ # 把mvn生成的xxx.jar移动到dockerfiles目录下 + - export IMAGE=`echo $IMAGE_NAME | sed 's/\//-/g'` + - cd dockerfiles && docker build -t $PROD_HARBOR/example/$IMAGE . + - docker login -u $PROD_HARBOR_USR -p $PROD_HARBOR_PWD $PROD_HARBOR + - docker push $PROD_HARBOR/example/$IMAGE + - docker logout $PROD_HARBOR + +job_push_prod_uat: ### 部署至阿里云uat环境 + stage: prod-uat-deploy + tags: + - prod-shell + when: manual + only: # 仅master和release分支可以执行该job + - master + - release + variables: + PROD_EXP_Domain: 'example-uat.xxxx.com' # job内部变量，指定该应用在uat环境的 ingress 域名 + script: + - export IMAGE=`echo $IMAGE_NAME | sed 's/\//-/g'` + - export PROJECT_NS=`echo $PROJECT_NS | sed 's/\//-/g'` + # 替换prod环境的参数配置 + - sed -i "s/PROJECT_NS/$PROJECT_NS/g" .ci/app.yaml + - sed -i "s/APP_NAME/$CI_PROJECT_NAME/g" .ci/app.yaml + - sed -i "s/APP_REP/1/g" .ci/app.yaml + - sed -i "s/AppDomain/$PROD_EXP_Domain/g" .ci/app.yaml + - sed -i "s/ProjectImage/$PROD_HARBOR\/example\/$IMAGE/g" .ci/app.yaml + - sed -i "s/DOCKER_KEY/$PROD_KEY/g" .ci/app.yaml + # + - mkdir -p /opt/kube/$PROJECT_NS/$APP_NAME + - cp -f .ci/app.yaml /opt/kube/$PROJECT_NS/$APP_NAME + - kubectl --kubeconfig=/etc/.aliyun/config apply -f .ci/app.yaml + +job_push_prod_release: ### 部署至阿里云正式环境 + stage: prod-deploy + tags: + - prod-shell + when: manual + only: # 仅master和release分支可以执行该job + - master + - release + variables: + PROD_EXP_Domain: 'example.xxxx.com' # 指定该应用在阿里云正式环境的 ingress 域名 + script: + - export IMAGE=`echo $IMAGE_NAME | sed 's/\//-/g'` + - export PROJECT_NS=`echo $PROJECT_NS | sed 's/\//-/g'` + # 替换prod环境的参数配置 + - sed -i "s/PROJECT_NS/$PROJECT_NS/g" .ci/app.yaml + - sed -i "s/APP_NAME/$CI_PROJECT_NAME/g" .ci/app.yaml + - sed -i "s/APP_REP/$PROD_APP_REP/g" .ci/app.yaml + - sed -i "s/AppDomain/$PROD_EXP_HOST/g" .ci/app.yaml + - sed -i "s/ProjectImage/$PROD_HARBOR\/example\/$IMAGE/g" .ci/app.yaml + - sed -i "s/DOCKER_KEY/$PROD_KEY/g" .ci/app.yaml + # + - mkdir -p /opt/kube/$PROJECT_NS/$APP_NAME + - cp -f .ci/app.yaml /opt/kube/$PROJECT_NS/$APP_NAME + - kubectl --kubeconfig=/etc/.aliyun/config apply -f .ci/app.yaml + +1/3 rollback: ### 定义生产环境回退job + stage: prod-rollback + tags: + - prod-shell + when: manual + only: + - master + - /^release.*$/ + variables: + PROJECT_NS: '$CI_PROJECT_NAMESPACE-prod-deploy' # 定义job内变量覆盖全局变量设置 + script: + - kubectl --kubeconfig=/etc/.aliyun/config -n $PROJECT_NS rollout undo deployment $CI_PROJECT_NAME --to-revision=1 + +2/3 rollback: + stage: prod-rollback + tags: + - prod-shell + when: manual + only: + - master + - /^release.*$/ + variables: + PROJECT_NS: '$CI_PROJECT_NAMESPACE-prod-deploy' # 定义job内变量覆盖全局变量设置 + script: + - kubectl --kubeconfig=/etc/.aliyun/config -n $PROJECT_NS rollout undo deployment $CI_PROJECT_NAME --to-revision=2 + +3/3 rollback: + stage: prod-rollback + tags: + - prod-shell + when: manual + only: + - master + - /^release.*$/ + variables: + PROJECT_NS: '$CI_PROJECT_NAMESPACE-prod-deploy' # 定义job内变量覆盖全局变量设置 + script: + - kubectl --kubeconfig=/etc/.aliyun/config -n $PROJECT_NS rollout undo deployment $CI_PROJECT_NAME --to-revision=3 +EOF +``` + +恭喜终于看完 gitlab-ci.yml 文件，怎么样，是不是一千个人可以写出一万个 CI/CD 流程 :) diff --git a/docs/guide/gitlab/gitlab-install.md b/docs/guide/gitlab/gitlab-install.md new file mode 100644 index 0000000..0e1b196 --- /dev/null +++ b/docs/guide/gitlab/gitlab-install.md @@ -0,0 +1,78 @@ +# 安装 gitlab + +gitlab 是深受企业用户喜爱的基于 git 的代码管理系统。安装 gitlab 最理想的方式是利用 gitlab charts 部署到 k8s 集群上，但此方式还未成熟，期待后续推出更成熟稳定版本；本文使用 Docker 方式安装 gitlab: + +- 环境：Ubuntu 16.04，虚机内存/CPU/存储请根据实际使用情况配置，一般`4C/8G/200G`足够 +- 安装 docker: 18.06.1-ce + +## 准备启动脚本 + +``` bash +$ cat > gitlab-setup.sh << EOF +#!/bin/bash +# 注意：设置 gitlab_shell_ssh_port 是为了后续可以使用 SSH 方式访问你的项目 +docker run --detach \\ + --hostname gitlab.test.com \\ + --env GITLAB_OMNIBUS_CONFIG="external_url 'http://gitlab.test.com/'; gitlab_rails['gitlab_shell_ssh_port'] = 6022;" \\ + --publish 443:443 --publish 80:80 --publish 6022:22 \\ + --name gitlab \\ + --restart always \\ + --volume /srv/gitlab/config:/etc/gitlab \\ + --volume /srv/gitlab/logs:/var/log/gitlab \\ + --volume /srv/gitlab/data:/var/opt/gitlab \\ + registry.docker-cn.com/gitlab/gitlab-ce:11.2.2-ce.0 +EOF +``` +执行启动脚本：`sh gitlab-setup.sh` 执行成功后，等待数分钟可以看到 + +``` +$ docker ps -a +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +4f9d5f97f494 registry.docker-cn.com/gitlab/gitlab-ce:11.2.2-ce.0 "/assets/wrapper" 9 minutes ago Up 9 minutes (healthy) 0.0.0.0:80->80/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:6022->22/tcp gitlab +``` +## 配置 gitlab + +``` +$ docker exec -it gitlab vi /etc/gitlab/gitlab.rb +``` +请阅读后修改（因为前面docker run 已经指定了必要参数，可以不修改，后续有需要再修改），修改保存以后需要重启容器 + +``` +$ docker restart gitlab +``` +## 首次访问 gitlab + +使用域名`gitlab.test.com`或者该主机 IP 首次登陆时会要求设置 root 用户的密码，完成后就可以用 root 和新设密码登陆；然后按需创建 Group, User, Projects等，还有相关配置。 + +## 备份数据 + +无论是企业、组织、个人都十分重视代码资产，之前我们的 gitlab 安装是单机版的，虽然可以有硬盘 raid 等保护，还有是丢失 gitlab 数据和配置的风险，因此我们有必要再做一些备份操作。这里利用 crontab 定期执行 rsync 命令备份到其他服务器。 + +``` bash +# 创建备份脚本 +cat > /root/gitlab-backup.sh << EOF +#!/bin/bash +# 请事先配置 gitlab 服务器到备份服务器的免密码 ssh 登陆 +rsync -av --delete /srv/gitlab/config '-e ssh -l root' 192.168.1.xx:/backup_gitlab/config +rsync -av --delete /srv/gitlab/data '-e ssh -l root' 192.168.1.xx:/backup_gitlab/data +EOF + +# 创建并应用 crontab +cat > /etc/cron.d/gitlab-backup << EOF +## 每3个小时同步备份一次，具体根据需要修改 +11 */3 * * * root bash /root/gitlab-backup.sh > /root/gitlab/sync.log 2>&1 +EOF +``` +如果 gitlab 服务器真的出现不可恢复的故障，丢失数据，那么至少保留有3小时前的备份，利用备份的文件，同样再用 docker 挂载 volume的方式运行，这样就可以恢复原 gitlab 服务运行。 + +## 升级 gitlab + +因为前面使用了 docker 方式安装，因此 gitlab 升级很方便。 + +- 升级前停止/删除容器：`$ docker stop gitlab && docker rm gitlab` +- 如上节执行备份数据 +- 修改 gitlab-setup.sh 指定新的版本，执行该脚本 + +## 参考 + +- 1.[Install GitLab with Docker](https://docs.gitlab.com/omnibus/docker/) diff --git a/docs/guide/gitlab/gitlab-runner.md b/docs/guide/gitlab/gitlab-runner.md new file mode 100644 index 0000000..2e08dff --- /dev/null +++ b/docs/guide/gitlab/gitlab-runner.md @@ -0,0 +1,65 @@ +## 安装 Gitlab Runner + +Gitlab Runner 安装方式有很多，可以参考官网文档 https://docs.gitlab.com/runner/install/; 这里为了方便直接在 Ubuntu1604 上 apt方式安装了。 + +``` bash +$ curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | sudo bash +$ apt-get install gitlab-runner +``` + +安装完成后就可以看到服务运行状态：`systemctl status gitlab-runner`，因为示例的java spring boot 项目需要，该虚机上要同时安装和配置 mvn 和 docker 环境。 + +注意：需要通过 gitlab-runner shell 执行docker镜像打包等命令，因此要修改下 gitlab-runner 服务运行用户："--user" "gitlab-runner" 改成 "--user" "root" + +``` bash +vi /etc/systemd/system/gitlab-runner.service +[Unit] +Description=GitLab Runner +After=syslog.target network.target +ConditionFileIsExecutable=/usr/lib/gitlab-runner/gitlab-runner + +[Service] +StartLimitInterval=5 +StartLimitBurst=10 +ExecStart=/usr/lib/gitlab-runner/gitlab-runner "run" "--working-directory" "/home/gitlab-runner" "--config" "/etc/gitlab-runner/config.toml" "--service" "gitlab-runner" "--syslog" "--user" "gitlab-runner" + +Restart=always +RestartSec=120 + +[Install] +WantedBy=multi-user.target +``` + +以上配置改完保存后执行服务重启： + +``` bash +$ systemctl daemon-reload +$ systemctl restart gitlab-runner +``` + +### 注册 Runner + +运行命令`gitlab-runner register`后进入交互式界面，按照提示注册，关注下面注释内容。 + +``` bash +$ gitlab-runner register +Runtime platform arch=amd64 os=linux pid=3269 revision=8bb608ff version=11.7.0 +Running in system-mode. + +Please enter the gitlab-ci coordinator URL (e.g. https://gitlab.com/): +http://gitlab.test.com/ ### 这里输入gitlab URL +Please enter the gitlab-ci token for this runner: +tzfBWCX-tQxxo1TCcoeJ ### 这里输入项目的token +Please enter the gitlab-ci description for this runner: +[k8s403]: build-shell ### 命名此runner +Please enter the gitlab-ci tags for this runner (comma separated): +build-shell ### 重要：指定runner tag，在gitlab-ci.yml文件中定义该tag才能执行 mvn编译/docker打包的任务 +Registering runner... succeeded runner=tzfBWCX- +Please enter the executor: docker-ssh, parallels, shell, ssh, virtualbox, kubernetes, docker, docker+machine, docker-ssh+machine: +shell ### 作为入门，在虚机上运行shell命令方式，方便调试 +Runner registered successfully. Feel free to start it, but if it's running already the config should be automatically reloaded! +``` +另外根据示例项目的ci/cd流程，还需要注册标签 tag 为 `beta-shell` 和 `prod-shell` 的两个 Runner; 注意这两个runner所在虚机需要分别配置测试k8s和生产k8s的 kubeconfig 配置，这样 Runner 才能通过 shell 脚本执行 kubectl apply 命令部署应用。三个 Runner 注册成功后可以看到如图： + +![active-runner](pics/active-runner.jpg) + diff --git a/docs/guide/gitlab/pics/active-runner.jpg b/docs/guide/gitlab/pics/active-runner.jpg new file mode 100644 index 0000000..e4782db Binary files /dev/null and b/docs/guide/gitlab/pics/active-runner.jpg differ diff --git a/docs/guide/gitlab/pics/cicd-pipeline.jpg b/docs/guide/gitlab/pics/cicd-pipeline.jpg new file mode 100644 index 0000000..4aab215 Binary files /dev/null and b/docs/guide/gitlab/pics/cicd-pipeline.jpg differ diff --git a/docs/guide/gitlab/pics/cicd-setting.jpg b/docs/guide/gitlab/pics/cicd-setting.jpg new file mode 100644 index 0000000..4cfa465 Binary files /dev/null and b/docs/guide/gitlab/pics/cicd-setting.jpg differ diff --git a/docs/guide/gitlab/pics/runner.jpg b/docs/guide/gitlab/pics/runner.jpg new file mode 100644 index 0000000..fc464d2 Binary files /dev/null and b/docs/guide/gitlab/pics/runner.jpg differ diff --git a/docs/guide/gitlab/readme.md b/docs/guide/gitlab/readme.md new file mode 100644 index 0000000..f2b5dd5 --- /dev/null +++ b/docs/guide/gitlab/readme.md @@ -0,0 +1,139 @@ +# Gitlab CI/CD 基础 + +Gitlab-ci 兼容 travis ci 格式，也是最流行的 CI 工具之一；本文讲解利用 gitlab, gitlab-runner, docker, harbor, kubernetes 等流行开源工具搭建一个自动化CI/CD流水线；示例配置以简单实用为原则，暂时没有选用 dind（docker in dockers）打包、gitlab Auto DevOps 等方式。一个最简单的流水线如下： + +- 代码提交 --> 镜像构建 --> 部署测试 --> 部署生产 + +## 0.前提条件 + +- 正常运行的 gitlab，[安装 gitlab 文档](gitlab-install.md) +- 正常运行的容器仓库，[安装 Harbor 文档](../harbor.md) +- 正常运行的 k8s，可以本地自建 k8s 集群，也可以使用公有云 k8s 集群 +- 若干虚机运行 gitlab-runner: 运行自动化流水线任务 pipeline job +- 了解代码管理流程 gitflow 等 + +## 1.准备测试项目代码 + +假设你要开发一个 spring boot 项目；先登陆你的 gitlab 账号，创建项目，上传你的代码；项目根目录看起来如下： + +``` +-rw-r--r-- 1 root root 44 Jan 2 16:38 eclipse.bat +drwxr-xr-x 8 root root 4096 Jan 7 15:29 .git/ +-rw-r--r-- 1 root root 276 Jan 7 08:44 .gitignore +drwxr-xr-x 3 root root 4096 Jan 7 08:44 example-api/ +drwxr-xr-x 3 root root 4096 Jan 7 08:44 example-biz/ +drwxr-xr-x 3 root root 4096 Jan 2 16:38 example-dal/ +drwxr-xr-x 3 root root 4096 Jan 2 16:38 example-web/ +-rw-r--r-- 1 root root 54 Jan 2 16:38 install.bat +-rw-r--r-- 1 root root 10419 Jan 2 16:38 pom.xml +``` +传统做法是在本地配置好相关环境后使用 mvn 编译生成jar包，然后测试运行jar；这里我们要把应用打包成 docker 镜像，并创建 CI/CD 流水线：如下示例，在项目根目录新增创建2个文件夹及相关文件 + +``` bash +dockerfiles ### 新增文件夹用来 docker 镜像打包 +└── Dockerfile # 定义 docker 镜像 +.ci ### 新增文件夹用来存放 CI/CD 相关内容 +├── app.yaml # k8s 平台的应用部署文件 +├── config.sh # 配置替换脚本 +└── gitlab-ci.yml # gitlab-ci 的主配置文件 +``` + +## 2.准备 docker 镜像描述文件 Dockerfile + +我们把 Dockerfile 放在独立目录下，java spring boot 应用可以这样写： + +``` bash +cat > dockerfiles/Dockerfile << EOF +FROM openjdk:8-jdk-alpine +VOLUME /tmp +COPY *.jar app.jar # 这里 *.jar 包就是后续在cicd pipeline 过程中 mvn 生成的jar包移动到此目录 +ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] +EOF +``` + +## 3.准备 CI/CD 相关脚本和文件 + +装完 gitlab 后使用浏览器登陆gitlab，很容易找到帮助文档，里面有介绍gitlab-ci的内容（文档权威、详细！请多多阅读~ 随着CI/CD流程的深入，部分内容也可以回来查阅），先看如下文档（假设你本地gitlab使用域名`gitlab.test.com`） + +- 文档首页 http://gitlab.test.com/help +- gitlab-ci 基本概念 http://gitlab.test.com/help/ci/README.md +- variables 变量 http://gitlab.test.com/help/ci/variables/README.md + +目录`.ci`下面的三个文件`app.yaml`, `config.sh`, `gitlab-ci.yml`是互相关联的；gitlab-ci.yml 文件中会调用到另外两个文件；文件之间又通过一些变量定义联系，流程中用到的变量大致可以分为三种： + +- 第一种是gitlab自身预定义变量（比如项目名: CI_PROJECT_NAME，流水线ID: CI_PIPELINE_ID）；无需更改； +- 第二种是在gitlab-ci.yml文件中定义的变量，一般是少量的自定义变量；按需少量改动； +- 第三种是用户可以在项目web界面配置的变量：“Settings”>"CI/CD">"Variables"，本示例项目用到该类型变量举例： + +|变量|值|注解| +|:-|:-|:-| +|BETA_APP_REP|1|beta环境应用副本数| +|BETA_DB_HOST|1.1.1.1:3306|beta环境应用连接数据库主机| +|BETA_DB_PWD|xxxx|beta环境数据库连接密码| +|BETA_DB_USR|xxxx|beta环境数据库连接用户| +|BETA_REDIS_HOST|1.1.1.2|beta环境redis主机| +|BETA_REDIS_PORT|6379|beta环境redis端口| +|BETA_REDIS_PWD|xxxx|beta环境redis密码| +|BETA_HARBOR|1.1.1.3|beta环境镜像仓库地址| +|BETA_HARBOR_PWD|xxxx|beta环境镜像仓库密码| +|BETA_HARBOR_USR|xxxx|beta环境镜像仓库用户| +|PROD_APP_REP|2|prod环境应用副本数| +|PROD_DB_HOST|2.2.2.1:3306|prod环境应用连接数据库主机| +|PROD_DB_PWD|xxxx|prod环境数据库连接密码| +|PROD_DB_USR|xxxx|prod环境数据库连接用户| +|PROD_REDIS_HOST|2.2.2.2|prod环境redis主机| +|PROD_REDIS_PORT|6379|prod环境redis端口| +|PROD_REDIS_PWD|xxxx|prod环境redis密码| +|PROD_HARBOR|2.2.2.3|prod环境镜像仓库地址| +|PROD_HARBOR_PWD|xxxx|prod环境镜像仓库密码| +|PROD_HARBOR_USR|xxxx|prod环境镜像仓库用户| +|...|...|根据项目需要自行添加设置| + +掌握了以上基础知识，可以开始以下三个任务： + +- 3.1[配置 gitlab-ci.yml](gitlab-ci.yml.md), 整个CI/CD的主配置文件，定义所有的CI/CD阶段和每个阶段的任务 +- 3.2[配置 config.sh](config.sh.md)，根据不同分支/环境替换不同的应用程序变量（对应上述第三种变量） +- 3.3[配置 app.yaml](app.yaml.md)，K8S应用部署简单模板，替换完成后可以部署到测试/生产的K8S平台上 + +## 4.为项目配置 CI/CD 及创建 RUNNER + +使用浏览器访问gitlab，登陆后，在项目页面进行配置，如图： + +![cicd-settings](pics/cicd-setting.jpg) + +- 在 General pipelines 中配置 Custom CI config path 为 .ci/gitlab-ci.yml +- 在 Variables 中配置需要用到的变量 +- 在 Runners 中配置注册 gitlab-runner 实例（runner 就是用来自动执行ci job的），点进去后如图： + +![runner](pics/runner.jpg) + +- 作为入门，先来手动创建 specific Runner，后续同样可以创建 Group Runners/Shared Runners，使用起来更方便；本文档暂不涉及在 kubernetes 自动创建 Runner + - 按照官网文档安装 Gitlab Runner，参考[文档](gitlab-runner.md) + - 记下 gitlab URL, 项目 token，注册 Runner 时要用到 + - 在 Gitlab Runner 注册本项目 + +## 5.提交代码测试 CI/CD Pipelines + +终于经过 1~4 步骤把示例项目的CI/CD 流水线创建出来了，是时候试试提交代码测试下成果了；别担心，初次 CI/CD job执行一般都会失败的:) 好在现在你已经基本了解了所有CI/CD流程与配置，失败了就查看错误日志一一排除。另外因为采用虚机安装 Runner 执行 shell 脚本的方式执行 ci job，我们始终可以登陆虚机以手动执行 shell 脚本的方式调试，这对于初学来说很有帮助。查看 CI/CD 执行情况如图： + +![cicd-pipeline](pics/cicd-pipeline.jpg) + +## 6.gitlab-ci 安全实践 + +现在为止 CICD Pipelines 已经可以跑通了，甚至稍微修改下 gitlab-ci.yml 配置，项目代码每一次提交后可以自动执行`编译`、`打包`、`部署测试`、`部署生产`等等工作；也许你还没来得及慢慢体会这顺畅的感觉，赶紧先踩个刹车，控制下车速；因为现在你需要考虑 gitlab-ci 的安全配置了，这很重要！ + +首先 gitlab 项目的基本安全就是项目成员控制，访问项目的权限分为：所有者（Owner），维护者（Maintainer），开发者（Developer），报告者（Reporter），访客（Guest）；详细的权限介绍请查阅官方文档，这里简单地介绍两类权限：所有者和维护者属于`特权用户`，开发者属于`普通用户`，他们应该具有如下权限区分： + +- 特权用户对整个项目负责，包括项目代码开发、配置管理、CI流程、测试环境、生产环境等 +- 特权用户可以提交代码到所有分支包括 master/release 分支，执行所有 ci job +- 普通用户只负责对应项目模块代码开发、不接触程序配置、只能访问测试环境 +- 普通用户只能提交代码到 develop/feature 分支，只能执行这两个分支的 ci job + +以下的安全实践配置作为个人经验分享，仅作参考；如果你的项目需要更高的安全性，请阅读 gitlab-ci 官方相关文档，尝试找到属于自己的最佳实践。 + +- 正确设置项目成员（Settings > Members），严格限制项目维护者（Maintainer）人数，大部分应该作为开发者（Developer）提交代码 +- 配置项目受保护分支/受保护标签，一般把master/release分支设置成受保护分支，限制只有维护者才能在保护分支commit和merge，从而限制只有维护者才能执行部署生产的 ci job，http://gitlab.test.com/help/user/project/protected_branches.md +- 配置受保护的变量，受保护的变量只在受保护分支和受保护tag的pipeline中可见，防止生产环境配置参数泄露，http://gitlab.test.com/help/ci/variables/README#protected-variables +- 配置受保护的Runner，只能执行受保护分支上的 ci jobs +- CICD Pipelines 中发布生产的任务请设置手动执行，同样生产的回退任务设置手动执行 + diff --git a/docs/guide/harbor.md b/docs/guide/harbor.md new file mode 100644 index 0000000..131b7bc --- /dev/null +++ b/docs/guide/harbor.md @@ -0,0 +1,175 @@ +## harbor + +Habor是由VMWare中国团队开源的容器镜像仓库。事实上，Habor是在Docker Registry上进行了相应的企业级扩展，从而获得了更加广泛的应用，这些新的企业级特性包括：管理用户界面，基于角色的访问控制 ，水平扩展，同步，AD/LDAP集成以及审计日志等。本文档仅说明部署单个基础harbor服务的步骤。 + +### 安装步骤 + +1. 在deploy节点下载最新的 [docker-compose](https://github.com/docker/compose/releases) 二进制文件，改名后把它放到项目 `/etc/ansible/bin`目录下（百度云的二进制文件中已包含） + +``` bash +wget https://github.com/docker/compose/releases/download/1.18.0/docker-compose-Linux-x86_64 +mv docker-compose-Linux-x86_64 /etc/ansible/bin/docker-compose +``` +2. 在deploy节点下载最新的 [harbor](https://github.com/vmware/harbor/releases) 离线安装包，把它放到项目 `/etc/ansible/down` 目录下，也可以从分享的百度云盘下载 + +3. 在deploy节点编辑/etc/ansible/hosts文件，可以参考 `example`目录下的模板，修改部分举例如下 + +``` bash +# 参数 NEW_INSTALL=(yes/no)：yes表示新建 harbor，并配置k8s节点的docker可以使用harbor仓库 +# no 表示仅配置k8s节点的docker使用已有的harbor仓库 +# 如果不需要设置域名访问 harbor，可以配置参数 HARBOR_DOMAIN="" +[harbor] +192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=yes +``` + +4. 在deploy节点执行 `ansible-playbook /etc/ansible/11.harbor.yml`，完成harbor安装和docker 客户端配置 + +### 安装讲解 + +根据 `11.harbor.yml`文件，harbor节点需要以下步骤： + +1. role `prepare` 基础系统环境准备 +1. role `docker` 安装docker +1. role `harbor` 安装harbor + +`kube-node`节点在harbor部署完之后，需要配置harbor的证书，并可以在hosts里面添加harbor的域名解析，如果你的环境中有dns服务器，可以跳过hosts文件设置 + +请在另外窗口打开 [roles/harbor/tasks/main.yml](../../roles/harbor/tasks/main.yml)，对照以下讲解 + +1. 下载docker-compose可执行文件到$PATH目录 +1. 自注册变量result判断是否已经安装harbor，避免重复安装问题 +1. 解压harbor离线安装包到指定目录 +1. 导入harbor所需 docker images +1. 创建harbor证书和私钥(复用集群的CA证书) +1. 修改harbor.cfg配置文件 +1. 启动harbor安装脚本 + +### 验证harbor + +1. 在harbor节点使用`docker ps -a` 查看harbor容器组件运行情况 +1. 浏览器访问harbor节点的IP地址 `https://$NodeIP`，使用账号 admin 和 密码 Harbor12345 (harbor.cfg 配置文件中的默认)登陆系统 + +### 在k8s集群使用harbor + +admin用户web登陆后可以方便的创建项目，并指定项目属性(公开或者私有)；然后创建用户，并在项目`成员`选项中选择用户和权限； + +#### 镜像上传 + +在node上使用harbor私有镜像仓库首先需要在指定目录配置harbor的CA证书，详见 `11.harbor.yml`文件。 + +使用docker客户端登陆`harbor.test.com`，然后把镜像tag成 `harbor.test.com/$项目名/$镜像名:$TAG` 之后，即可使用docker push 上传 + +``` bash +docker login harbor.test.com +Username: +Password: +Login Succeeded +docker tag busybox:latest harbor.test.com/library/busybox:latest +docker push harbor.test.com/library/busybox:latest +The push refers to a repository [harbor.test.com/library/busybox] +0271b8eebde3: Pushed +latest: digest: sha256:91ef6c1c52b166be02645b8efee30d1ee65362024f7da41c404681561734c465 size: 527 +``` +#### k8s中使用harbor + +1. 如果镜像保存在harbor中的公开项目中，那么只需要在yaml文件中简单指定harbor私有镜像即可，例如 + +``` bash +apiVersion: v1 +kind: Pod +metadata: + name: test-busybox +spec: + containers: + - name: test-busybox + image: harbor.test.com/xxx/busybox:latest + imagePullPolicy: Always +``` + +2. 如果镜像保存在harbor中的私有项目中，那么yaml文件中使用该私有项目的镜像需要指定`imagePullSecrets`，例如 + +``` bash +apiVersion: v1 +kind: Pod +metadata: + name: test-busybox +spec: + containers: + - name: test-busybox + image: harbor.test.com/xxx/busybox:latest + imagePullPolicy: Always + imagePullSecrets: + - name: harborkey1 +``` +其中 `harborKey1`可以用以下两种方式生成： + ++ 1.使用 `kubectl create secret docker-registry harborkey1 --docker-server=harbor.test.com --docker-username=admin --docker-password=Harbor12345 --docker-email=team@test.com` ++ 2.使用yaml配置文件生成 + +``` bash +//harborkey1.yaml +apiVersion: v1 +kind: Secret +metadata: + name: harborkey1 + namespace: default +data: + .dockerconfigjson: {base64 -w 0 ~/.docker/config.json} +type: kubernetes.io/dockerconfigjson +``` +前面docker login会在~/.docker下面创建一个config.json文件保存鉴权串，这里secret yaml的.dockerconfigjson后面的数据就是那个json文件的base64编码输出（-w 0让base64输出在单行上，避免折行） + +### 管理harbor + ++ 日志目录 `/var/log/harbor` ++ 数据目录 `/data` ，其中最主要是 `/data/database` 和 `/data/registry` 目录，如果你要彻底重新安装harbor，删除这两个目录即可 + +先进入harbor安装目录 `cd /data/harbor`，常规操作如下： + +1. 暂停harbor `docker-compose stop` : docker容器stop，并不删除容器 +2. 恢复harbor `docker-compose start` : 恢复docker容器运行 +3. 停止harbor `docker-compose down -v` : 停止并删除docker容器 +4. 启动harbor `docker-compose up -d` : 启动所有docker容器 + +修改harbor的运行配置，需要如下步骤： + +``` bash +# 停止 harbor + docker-compose down -v +# 修改配置 + vim harbor.cfg +# 执行./prepare已更新配置到docker-compose.yml文件 + ./prepare +# 启动 harbor + docker-compose up -d +``` +#### harbor 升级 + +以下步骤基于harbor 1.1.2 版本升级到 1.2.2版本 + +``` bash +# 进入harbor解压缩后的目录，停止harbor +cd /data/harbor +docker-compose down + +# 备份这个目录 +cd .. +mkdir -p /backup && mv harbor /backup/harbor + +# 下载更新的离线安装包，并解压 +tar zxvf harbor-offline-installer-v1.2.2.tgz -C /data + +# 使用官方数据库迁移工具，备份数据库，修改数据库连接用户和密码，创建数据库备份目录 +# 迁移工具使用docker镜像，镜像tag由待升级到目标harbor版本决定，这里由 1.1.2升级到1.2.2，所以使用 tag 1.2 +docker pull vmware/harbor-db-migrator:1.2 +mkdir -p /backup/db-1.1.2 +docker run -it --rm -e DB_USR=root -e DB_PWD=xxxx -v /data/database:/var/lib/mysql -v /backup/db-1.1.2:/harbor-migration/backup vmware/harbor-db-migrator:1.2 backup + +# 因为新老版本数据库结构不一样，需要数据库migration +docker run -it --rm -e DB_USR=root -e DB_PWD=xxxx -v /data/database:/var/lib/mysql vmware/harbor-db-migrator:1.2 up head + +# 修改新版本 harbor.cfg配置，需要保持与老版本相关配置项保持一致，然后执行安装即可 +cd /data/harbor +vi harbor.cfg +./install.sh +``` diff --git a/docs/guide/heapster.1.4.3.md b/docs/guide/heapster.1.4.3.md new file mode 100644 index 0000000..982c72e --- /dev/null +++ b/docs/guide/heapster.1.4.3.md @@ -0,0 +1,91 @@ +## heapster + +`Heapster` 监控整个集群资源的过程：首先kubelet内置的cAdvisor收集本node节点的容器资源占用情况，然后heapster从kubelet提供的api采集节点和容器的资源占用，最后heapster 持久化数据存储到`influxdb`中（也可以是其他的存储后端,Google Cloud Monitoring等）。 + +`Grafana` 则通过配置数据源指向上述 `influxdb`，从而界面化显示监控信息。 + +### 部署 + +访问 [heapster release](https://github.com/kubernetes/heapster)页面下载最新 release 1.4.3，参考目录`heapster-1.3.0/deploy/kube-config/influxdb`，因为这个官方release 在k8s1.8.4使用还是有不少问题，请在参考的基础上使用本项目提供的yaml文件 + +1. [grafana](../../manifests/heapster/grafana.yaml) +1. [heapster](../../manifests/heapster/heapster.yaml) +1. [influxdb](../../manifests/heapster/influxdb.yaml) + +安装比较简单 `kubectl create -f /etc/ansible/manifests/heapster/`，主要讲一下注意事项 + +#### grafana.yaml配置 + ++ 修改`heapster-grafana-amd64`镜像，v4.2.0版本修改成 v4.4.3版本，否则 grafana pod无法起来，报`CrashLoopBackOff`错误，详见[ISSUE](https://github.com/kubernetes/heapster/issues/1806) ++ 参数`- name: GF_SERVER_ROOT_URL`的设置要根据后续访问grafana的方式确定，如果使用 NodePort方式访问，必须设置成:`value: /`；如果使用apiserver proxy方式，必须设置成`value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy/`，注意官方文件中预设的`value: /api/v1/proxy/namespaces/kube-system/services/monitoring-grafana/`已经不适合k8s 1.8.0版本了， ++ `kubernetes.io/cluster-service: 'true'` 和 `type: NodePort` 根据上述的访问方式设置，建议使用apiserver 方式，可以增加安全控制 + +#### heapster.yaml配置 + ++ 需要配置 RBAC 把 ServiceAccount `heapster` 与集群预定义的集群角色 `system:heapster` 绑定，这样heapster pod才有相应权限去访问 apiserver + +#### influxdb.yaml配置 + ++ influxdb 官方建议使用命令行或 HTTP API 接口来查询数据库，从 v1.1.0 版本开始默认关闭 admin UI，这里参考[opsnull](https://github.com/opsnull/follow-me-install-kubernetes-cluster/blob/master/10-%E9%83%A8%E7%BD%B2Heapster%E6%8F%92%E4%BB%B6.md)给出的方法，增加ConfigMap配置，然后挂载到容器中，覆盖默认配置 ++ 注意influxdb 这个版本只能使用 NodePort方式访问它的admin UI，才能正确连接数据库 + +### 验证 + +``` bash +$ kubectl get pods -n kube-system | grep -E 'heapster|monitoring' +heapster-3273315324-tmxbg 1/1 Running 0 11m +monitoring-grafana-2255110352-94lpn 1/1 Running 0 11m +monitoring-influxdb-884893134-3vb6n 1/1 Running 0 11m +``` +扩展检查Pods日志： +``` bash +$ kubectl logs heapster-3273315324-tmxbg -n kube-system +$ kubectl logs monitoring-grafana-2255110352-94lpn -n kube-system +$ kubectl logs monitoring-influxdb-884893134-3vb6n -n kube-system +``` +部署完heapster，使用上一步介绍方法查看kubernets dashboard 界面，就可以看到各 Nodes、Pods 的 CPU、内存、负载等利用率曲线图，如果 dashboard上还无法看到利用率图，使用以下命令重启 dashboard pod： ++ 首先删除 `kubectl scale deploy kubernetes-dashboard --replicas=0 -n kube-system` ++ 然后新建 `kubectl scale deploy kubernetes-dashboard --replicas=1 -n kube-system` + +### 访问 grafana + +#### 1.通过apiserver 访问（建议的方式） + +``` bash +kubectl cluster-info | grep grafana +monitoring-grafana is running at https://x.x.x.x:6443/api/v1/namespaces/kube-system/services/monitoring-grafana/proxy +``` +请参考上一步 [访问dashboard](dashboard.md)同样的方式，使用证书或者密码认证（参照hosts文件配置，默认：用户admin 密码test1234），访问`https://x.x.x.x:6443/api/v1/namespaces/kube-system/services/monitoring-grafana/proxy`即可，如图可以点击[Home]选择查看 `Cluster` `Pods`的监控图形 + +![grafana](../../pics/grafana.png) + +#### 2.通过NodePort 访问 + ++ 修改 `Service` 允许 type: NodePort ++ 修改 `Deployment`中参数`- name: GF_SERVER_ROOT_URL`为 `value: /` ++ 如果之前grafana已经运行，使用 `kubectl replace --force -f /etc/ansible/manifests/heapster/grafana.yaml` 重启 grafana插件 + +``` bash +kubectl get svc -n kube-system|grep grafana +monitoring-grafana NodePort 10.68.135.50 80:5855/TCP 11m +``` +然后用浏览器访问 http://NodeIP:5855 + +### 访问 influxdb + +官方建议使用命令行或 HTTP API 接口来查询`influxdb`数据库，如非必要就跳过此步骤 + +目前根据测试 k8s v1.8.4 使用 NodePort 方式访问 admin 界面后才能正常连接数据库 + +``` bash +kubectl get svc -n kube-system|grep influxdb +monitoring-influxdb NodePort 10.68.195.193 8086:3382/TCP,8083:7651/TCP 12h +``` ++ 如上例子，8083是管理页面端口，对外暴露的端口为7651 ++ 8086 是数据连接端口，对外暴露的端口为3382 + +使用浏览器访问 http://NodeIP:7651，如图在页面的 “Connection Settings” 的 Host 中输入 node IP， Port 中输入 3382(由8086对外暴露的端口)，点击 “Save” 即可 + +![influxdb](../../pics/influxdb.png) + + diff --git a/docs/guide/heapster.md b/docs/guide/heapster.md new file mode 100644 index 0000000..a20e7e2 --- /dev/null +++ b/docs/guide/heapster.md @@ -0,0 +1,167 @@ +## 第一部分：heapster + ++ 本文档基于heapster 1.5.1和k8s 1.9.x，旧版文档请看[heapster 1.4.3](heapster.1.4.3.md) + +`Heapster` 监控整个集群资源的过程：首先kubelet内置的cAdvisor收集本node节点的容器资源占用情况，然后heapster从kubelet提供的api采集节点和容器的资源占用，最后heapster 持久化数据存储到`influxdb`中（也可以是其他的存储后端,Google Cloud Monitoring等）。 + +`Grafana` 则通过配置数据源指向上述 `influxdb`，从而界面化显示监控信息。 + +### 部署 + +访问 [heapster release](https://github.com/kubernetes/heapster)页面下载最新 release 1.5.1，参考目录`heapster-1.5.1/deploy/kube-config/influxdb`，请在参考官方yaml文件的基础上使用本项目提供的yaml文件 + +1. [grafana](../../manifests/heapster/grafana.yaml) +1. [heapster](../../manifests/heapster/heapster.yaml) +1. [influxdb](../../manifests/heapster/influxdb.yaml) + +安装比较简单 `kubectl create -f /etc/ansible/manifests/heapster/`，主要讲一下注意事项 + +#### grafana.yaml配置 + ++ 参数`- name: GF_SERVER_ROOT_URL`的设置要根据后续访问grafana的方式确定，如果使用 NodePort方式访问，必须设置成:`value: /`；如果使用apiserver proxy方式，必须设置成`value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy/` ++ `kubernetes.io/cluster-service: 'true'` 和 `type: NodePort` 根据上述的访问方式设置，建议使用apiserver 方式，可以增加安全控制 + +#### heapster.yaml配置 + ++ 需要配置 RBAC 把 ServiceAccount `heapster` 与集群预定义的集群角色 `system:heapster` 绑定，这样heapster pod才有相应权限去访问 apiserver + +#### influxdb.yaml配置 + ++ influxdb 官方建议使用命令行或 HTTP API 接口来查询数据库，从 v1.1.0 版本开始默认关闭 admin UI, 从 v1.3.3 版本开始已经移除 admin UI 插件，如果你因特殊原因需要访问admin UI，请使用 v1.1.1 版本并使用configMap 配置开启它。参考[heapster 1.4.3](heapster.1.4.3.md)，具体配置yaml文件参考[influxdb v1.1.1](../../manifests/heapster/influxdb-v1.1.1/influxdb.yaml) + +### 验证 + +``` bash +$ kubectl get pods -n kube-system | grep -E 'heapster|monitoring' +heapster-3273315324-tmxbg 1/1 Running 0 11m +monitoring-grafana-2255110352-94lpn 1/1 Running 0 11m +monitoring-influxdb-884893134-3vb6n 1/1 Running 0 11m +``` +检查Pods日志： +``` bash +$ kubectl logs heapster-3273315324-tmxbg -n kube-system +$ kubectl logs monitoring-grafana-2255110352-94lpn -n kube-system +$ kubectl logs monitoring-influxdb-884893134-3vb6n -n kube-system +``` +部署完heapster，使用上一步介绍方法查看kubernets dashboard 界面，就可以看到各 Nodes、Pods 的 CPU、内存、负载等利用率曲线图，如果 dashboard上还无法看到利用率图，使用以下命令重启 dashboard pod： ++ 首先删除 `kubectl scale deploy kubernetes-dashboard --replicas=0 -n kube-system` ++ 然后新建 `kubectl scale deploy kubernetes-dashboard --replicas=1 -n kube-system` + +部署完heapster，直接使用 `kubectl` 客户端工具查看资源使用 + +``` bash +# 查看node 节点资源使用情况 +$ kubectl top node +# 查看各pod 的资源使用情况 +$ kubectl top pod --all-namespaces +``` + +### 访问 grafana + +#### 1.通过apiserver 访问（建议的方式） + +``` bash +kubectl cluster-info | grep grafana +monitoring-grafana is running at https://x.x.x.x:6443/api/v1/namespaces/kube-system/services/monitoring-grafana/proxy +``` +请参考上一步 [访问dashboard](dashboard.md)同样的方式，使用证书或者密码认证（参照hosts文件配置，默认：用户admin 密码test1234），访问`https://x.x.x.x:6443/api/v1/namespaces/kube-system/services/monitoring-grafana/proxy`即可，如图可以点击[Home]选择查看 `Cluster` `Pods`的监控图形 + +![grafana](../../pics/grafana.png) + +#### 2.通过NodePort 访问 + ++ 修改 `Service` 允许 type: NodePort ++ 修改 `Deployment`中参数`- name: GF_SERVER_ROOT_URL`为 `value: /` ++ 如果之前grafana已经运行，使用 `kubectl replace --force -f /etc/ansible/manifests/heapster/grafana.yaml` 重启 grafana插件 + +``` bash +kubectl get svc -n kube-system|grep grafana +monitoring-grafana NodePort 10.68.135.50 80:5855/TCP 11m +``` +然后用浏览器访问 http://NodeIP:5855 + + +## 第二部分：heapster 之监控数据持久化 + +我们知道监控数据是存储到`influxdb`中的，但是默认情况下`influxdb.yaml`文件中存储使用的是`emptyDir`类型，所以当influxdb POD被删除时，监控数据也就丢失了，以下是使用nfs持久化保存监控数据的例子。 + +### 前提 +环境准备一个nfs服务器，如果没有可以参考[nfs-server](nfs-server.md)创建。 + +### 创建 PV +PersistentVolume (PV) 和 PersistentVolumeClaim (PVC) 提供了方便的持久化卷；`PV` 是集群的存储资源，就像 `node`是集群的计算资源，PV 可以静态或动态创建，这里使用静态方式创建；`PVC` 就是用来申请`PV` 资源，它可以直接挂载在`POD` 里面使用。更多知识请访问[官网](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)。根据你监控日志的多少和需保存时间需求创>建固定大小的`PV` 资源，例子： + +``` bash +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-influxdb +spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteMany + volumeMode: Filesystem + persistentVolumeReclaimPolicy: Recycle + storageClassName: slow + nfs: + # 根据实际共享目录修改 + path: /share + # 根据实际 nfs服务器地址修改 + server: 192.168.1.208 +``` + +### 修改influxdb 存储卷 +使用PVC 替换 volumes `emptyDir:{}`，创建PVC 如下： + +``` bash +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: influxdb-claim + namespace: kube-system +spec: + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 3Gi + storageClassName: slow +``` ++ 注意`PV` 是不区分namespace，而`PVC` 是区分namespace的 + +### 安装持久化influxdb + +如果之前已经安装本项目创建了`heapster`，请使用如下删除 `influxdb POD`: + +``` bash +kubectl delete -f /etc/ansible/manifests/heapster/influxdb.yaml +``` + +然后使用如下命令新建持久化的 `influxdb POD` : + +``` bash +kubectl create -f /etc/ansible/manifests/heapster/influxdb-with-pv/ +``` + +### 验证监控数据的持久性 + ++ 1.查看集群 `pv` `pvc` 情况 + +``` bash +$ kubectl get pv +$ kubectl get pvc --all-namespaces +``` + ++ 2.手动删除 `influxdb`，半小时后再次创建，登陆grafana 确认历史数据是否还在。 + +``` bash +# 删除 influxdb deploy +kubectl delete -f /etc/ansible/manifests/heapster/influxdb-with-pv/influxdb.yaml + +# 等待半小时后重新创建 +kubectl create -f /etc/ansible/manifests/heapster/influxdb-with-pv/influxdb.yaml +``` + + diff --git a/docs/guide/helm.md b/docs/guide/helm.md new file mode 100644 index 0000000..e221b01 --- /dev/null +++ b/docs/guide/helm.md @@ -0,0 +1,58 @@ +# Helm + +`Helm`致力于成为k8s集群的应用包管理工具，希望像linux 系统的`RPM` `DPKG`那样成功；确实在k8s上部署复杂一点的应用很麻烦，需要管理很多yaml文件（configmap,controller,service,rbac,pv,pvc等等），而helm能够整齐管理这些文档：版本控制，参数化安装，方便的打包与分享等。 +- 建议积累一定k8s经验以后再去使用helm；对于初学者来说手工去配置那些yaml文件对于快速学习k8s的设计理念和运行原理非常有帮助，而不是直接去使用helm，面对又一层封装与复杂度。 +- 本文参考 helm 官网安全实践启用 TLS 认证，参考 https://docs.helm.sh/using_helm/#securing-your-helm-installation + +## 安全安装 helm（在线） + +在helm客户端和tiller服务器间建立安全的SSL/TLS认证机制；tiller服务器和helm客户端都是使用同一CA签发的`client cert`，然后互相识别对方身份。建议通过本项目提供的`ansible role`安装，符合官网上介绍的安全加固措施，在delpoy节点运行: +``` bash +# 1.如果已安装非安全模式，使用 helm reset 清理 +# 2.配置默认helm参数 vi /etc/ansible/roles/helm/defaults/main.yml +# 3.执行安装 +$ ansible-playbook /etc/ansible/roles/helm/helm.yml +``` + +简单介绍下`/roles/helm/tasks/main.yml`中的步骤 + +- 1-下载最新release的helm客户端到/etc/ansible/bin目录下，再由它自动推送到deploy的{{ bin_dir }}目录下 +- 2-由集群CA签发helm客户端证书和私钥 +- 3-由集群CA签发tiller服务端证书和私钥 +- 4-创建tiller专用的RBAC配置，只允许helm在指定的namespace查看和安装应用 +- 5-安全安装tiller到集群，tiller服务启用tls验证 +- 6-配置helm客户端使用tls方式与tiller服务端通讯 + +### 注意因使用了TLS认证，所以helm命令执行分以下两种情况 + +- 执行与tiller服务有关的命令，比如 `helm ls` `helm version` `helm install`等需要加`--tls`参数 +- 执行其他命令，比如`helm search` `helm fetch` `helm home`等不需要加`--tls` +- helm v2.11.0及以上版本，启用环境变量 HELM_TLS_ENABLE=true，可以都不用加 --tls 参数 + +## 安全安装 helm（离线） +在内网环境中，由于不能访问互联网，无法连接repo地址，使用上述的在线安装helm的方式会报错。因此需要使用离线安装的方法来安装。 +离线安装步骤： +```bash +# 1.创建本地repo +mkdir -p /opt/helm-repo +# 2.启动helm repo server,如果要其他服务器访问，改为本地IP +nohup helm serve --address 127.0.0.1:8879 --repo-path /opt/helm-repo & +# 3.更改helm 配置文件 +将/etc/ansible/role/helm/default/main.yml中repo的地址改为 http://127.0.0.1:8879 +cat </etc/ansible/role/helm/default/main.yml +helm_namespace: kube-system +helm_cert_cn: helm001 +tiller_sa: tiller +tiller_cert_cn: tiller001 +tiller_image: jmgao1983/tiller:v2.9.1 +#repo_url: https://kubernetes-charts.storage.googleapis.com +repo_url: http://127.0.0.1:8879 +# 如果默认官方repo 网络访问不稳定可以使用如下的阿里云镜像repo +#repo_url: https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts +EOF +# 4.运行安全helm命令 +ansible-playbook /etc/ansible/role/helm/helm.yml +``` +## 使用helm安装应用到k8s上 + +请阅读本项目文档[helm安装prometheus监控](prometheus.md) diff --git a/docs/guide/hpa.md b/docs/guide/hpa.md new file mode 100644 index 0000000..cebb7f1 --- /dev/null +++ b/docs/guide/hpa.md @@ -0,0 +1,56 @@ +## Horizontal Pod Autoscaling + +自动水平伸缩，是指运行在k8s上的应用负载(POD)，可以根据资源使用率进行自动扩容、缩容；我们知道应用的资源使用率通常都有高峰和低谷，所以k8s的`HPA`特性应运而生；它也是最能体现区别于传统运维的优势之一，不仅能够弹性伸缩，而且完全自动化！ + +根据 CPU 使用率或自定义 metrics 自动扩展 Pod 数量（支持 replication controller、deployment）；k8s1.6版本之前是通过kubelet来获取监控指标，1.6版本之后是通过api server、heapster或者kube-aggregator来获取监控指标。 + +### Metrics支持 + +根据不同版本的API中，HPA autoscale时靠以下指标来判断资源使用率： +- autoscaling/v1: CPU +- autoscaling/v2alpha1 + - 内存 + - 自定义metrics + - 多metrics组合: 根据每个metric的值计算出scale的值，并将最大的那个值作为扩容的最终结果 + +### 基础示例 + +本实验环境基于k8s 1.8 和 1.9，仅使用`autoscaling/v1` 版本API，**注意确保**`k8s` 集群插件`kubedns` 和 `heapster` 工作正常。 + +``` bash +# 创建deploy和service +$ kubectl run php-apache --image=pilchard/hpa-example --requests=cpu=200m --expose --port=80 + +# 创建autoscaler +$ kubectl autoscale deploy php-apache --cpu-percent=50 --min=1 --max=10 + +# 等待3~5分钟查看hpa状态 +$ kubectl get hpa php-apache +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE +php-apache Deployment/php-apache 0% / 50% 1 10 1 3m + +# 增加负载 +$ kubectl run --rm -it load-generator --image=busybox /bin/sh +Hit enter for command prompt +$ while true; do wget -q -O- http://php-apache; done; + +# 等待约5分钟查看hpa显示负载增加，且副本数目增加为4 +$ kubectl get hpa php-apache +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE +php-apache Deployment/php-apache 430% / 50% 1 10 4 4m + +# 注意k8s为了避免频繁增删pod，对副本的增加速度有限制 +# 实验过程可以看到副本数目从1到4到8到10，大概都需要4~5分钟的缓冲期 +$ kubectl get hpa php-apache +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE +php-apache Deployment/php-apache 86% / 50% 1 10 8 9m +$ kubectl get hpa php-apache +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE +php-apache Deployment/php-apache 52% / 50% 1 10 10 12m + +# 清除负载，CTRL+C 结束上述循环程序，稍后副本数目变回1 +$ kubectl get hpa php-apache +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE +php-apache Deployment/php-apache 0% / 50% 1 10 1 17m +``` + diff --git a/docs/guide/index.md b/docs/guide/index.md new file mode 100644 index 0000000..22a6519 --- /dev/null +++ b/docs/guide/index.md @@ -0,0 +1,22 @@ +## 使用指南 + +### 附加组件安装 + +- 安装 [kubedns](kubedns.md) +- 安装 [dashboard](dashboard.md) +- 安装 [metrics-server](metrics-server.md) +- 安装 [prometheus](prometheus.md) +- 安装 [heapster](heapster.md) DEPRECATED WARNNING +- 安装 [ingress](ingress.md) +- 安装 [helm](helm.md) +- 安装 [efk](efk.md) +- 安装 [harbor](harbor.md) +- 安装 [metallb](metallb.md) + +### 基础特性演示 + +- 自动水平伸缩 [Horizontal Pod Autoscaling](hpa.md) +- 网络安全策略 [Network Policy](networkpolicy.md) +- 滚动更新 [rollingupdate](rollingupdateWithZeroDowntime.md) + + diff --git a/docs/guide/ingress-tls.md b/docs/guide/ingress-tls.md new file mode 100644 index 0000000..5e44173 --- /dev/null +++ b/docs/guide/ingress-tls.md @@ -0,0 +1,106 @@ +# 使用 traefik 配置 https ingress + +本文档基于 traefik 配置 https ingress 规则，请先阅读[配置基本 ingress](ingress.md)。与基本 ingress-controller 相比，需要额外配置 https tls 证书，主要步骤如下： + +## 1.准备 tls 证书 + +可以使用Let's Encrypt签发的免费证书，这里为了测试方便使用自签证书 (tls.key/tls.crt)，注意CN 配置为 ingress 的域名： + +``` bash +$ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=hello.test.com" +``` + +## 2.在 kube-system 命名空间创建 secret: traefik-cert，以便后面 traefik-controller 挂载该证书 + +``` bash +$ kubectl -n kube-system create secret tls traefik-cert --key=tls.key --cert=tls.crt +``` + +## 3.创建 traefik-controller，增加 traefik.toml 配置文件及https 端口暴露等，详见该 yaml 文件 + +``` bash +$ kubectl apply -f /etc/ansible/manifests/ingress/traefik/tls/traefik-controller.yaml +``` + +## 4.创建 https ingress 例子 + +``` bash +# 创建示例应用 +$ kubectl run test-hello --image=nginx --port=80 --expose +# hello-tls-ingress 示例 +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: hello-tls-ingress + annotations: + kubernetes.io/ingress.class: traefik +spec: + rules: + - host: hello.test.com + http: + paths: + - backend: + serviceName: test-hello + servicePort: 80 + tls: + - secretName: traefik-cert +# 创建https ingress +$ kubectl apply -f /etc/ansible/manifests/ingress/traefik/tls/hello-tls.ing.yaml +# 注意根据hello示例，需要在default命名空间创建对应的secret: traefik-cert +$ kubectl create secret tls traefik-cert --key=tls.key --cert=tls.crt +``` + +## 5.验证 https 访问 + +验证 traefik-ingress svc + +``` bash +$ kubectl get svc -n kube-system traefik-ingress-service +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +traefik-ingress-service NodePort 10.68.250.253 80:23456/TCP,443:23457/TCP,8080:35941/TCP 66m +``` + +可以看到项目默认使用nodePort 23456暴露traefik 80端口，nodePort 23457暴露 traefik 443端口，因此在客户端 hosts 增加记录 `$Node_IP hello.test.com`之后，可以在浏览器验证访问如下： + +``` bash +https://hello.test.com:23457 +``` + +如果你已经配置了[转发 ingress nodePort](../op/loadballance_ingress_nodeport.md)，那么增加对应 hosts记录后，可以验证访问 `https://hello.test.com` + +## 配置 dashboard ingress + +前提1：k8s 集群的dashboard 已安装 + +``` +$ kubectl get svc -n kube-system | grep dashboard +kubernetes-dashboard NodePort 10.68.211.168 443:39308/TCP 3d11h +``` +前提2：`/etc/ansible/manifests/ingress/traefik/tls/traefik-controller.yaml`的配置文件`traefik.toml`开启了`insecureSkipVerify = true` + +配置 dashboard ingress：`kubectl apply -f /etc/ansible/manifests/ingress/traefik/tls/k8s-dashboard.ing.yaml` 内容如下： + +``` +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: kubernetes-dashboard + namespace: kube-system + annotations: + traefik.ingress.kubernetes.io/redirect-entry-point: https +spec: + rules: + - host: dashboard.test.com + http: + paths: + - path: / + backend: + serviceName: kubernetes-dashboard + servicePort: 443 +``` +- 注意annotations 配置了 http 跳转 https 功能 +- 注意后端服务是443端口 + +## 参考 + +- [Add a TLS Certificate to the Ingress](https://docs.traefik.io/user-guide/kubernetes/#add-a-tls-certificate-to-the-ingress) diff --git a/docs/guide/ingress.md b/docs/guide/ingress.md new file mode 100644 index 0000000..8bd5c0f --- /dev/null +++ b/docs/guide/ingress.md @@ -0,0 +1,144 @@ +## Ingress简介 + +ingress就是从外部访问k8s集群的入口，将用户的URL请求转发到不同的service上。ingress相当于nginx反向代理服务器，它包括的规则定义就是URL的路由信息；它的实现需要部署`Ingress controller`(比如 [traefik](https://github.com/containous/traefik) [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 等)，`Ingress controller`通过apiserver监听ingress和service的变化，并根据规则配置负载均衡并提供访问入口，达到服务发现的作用。 + +- 未配置ingress： + +集群外部 -> NodePort -> K8S Service + +- 配置ingress: + +集群外部 -> Ingress -> K8S Service + +- **注意：ingress 本身也需要部署`Ingress controller`时使用以下几种方式让外部访问** + - 使用`NodePort`方式 + - 使用`hostPort`方式 + - 使用LoadBalancer地址方式 + +- 以下讲解基于`Traefik`，如果想要了解`ingress-nginx`的原理与实践，推荐阅读博客[烂泥行天下](https://www.ilanni.com/?p=14501)的相关文章 + +### 部署 Traefik + +Traefik 提供了一个简单好用 `Ingress controller`，下文侧重讲解 ingress部署和测试例子。请查看yaml配置 [traefik-ingress.yaml](../../manifests/ingress/traefik/traefik-ingress.yaml)，参考[traefik 官方k8s例子](https://github.com/containous/traefik/tree/master/examples/k8s) + +#### 安装 traefik ingress-controller + +``` bash +kubectl create -f /etc/ansible/manifests/ingress/traefik/traefik-ingress.yaml +``` ++ 注意需要配置 `RBAC`授权 ++ 注意`trafik pod`中 `80`端口为 traefik ingress-controller的服务端口，`8080`端口为 traefik 的管理WEB界面；为后续配置方便指定`80` 端口暴露`NodePort`端口为 `23456`(对应于在hosts配置中`NODE_PORT_RANGE`范围内可用端口) + +#### 验证 traefik ingress-controller + +``` bash +# kubectl get deploy -n kube-system traefik-ingress-controller +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +traefik-ingress-controller 1 1 1 1 4m + +# kubectl get svc -n kube-system traefik-ingress-service +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +traefik-ingress-service NodePort 10.68.69.170 80:23456/TCP,8080:34815/TCP 4m +``` ++ 可以看到`traefik-ingress-service` 服务端口`80`暴露的nodePort确实为`23456` + +#### 测试 ingress + ++ 首先创建测试用K8S应用，并且该应用服务不用nodePort暴露，而是用ingress方式让外部访问 + +``` bash +kubectl run test-hello --image=nginx --expose --port=80 +## +# kubectl get deploy test-hello +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +test-hello 1 1 1 1 56s +# kubectl get svc test-hello +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +test-hello ClusterIP 10.68.124.115 80/TCP 1m +``` ++ 然后为这个应用创建 ingress，`kubectl create -f /etc/ansible/manifests/ingress/test-hello.ing.yaml` + +``` bash +# test-hello.ing.yaml内容 +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: test-hello +spec: + rules: + - host: hello.test.com + http: + paths: + - path: / + backend: + serviceName: test-hello + servicePort: 80 +``` ++ 集群内部尝试访问: `curl -H Host:hello.test.com 10.68.69.170(traefik-ingress-service的服务地址)` 能够看到欢迎页面 `Welcome to nginx!`； ++ 在集群外部尝试访问(假定集群一个NodeIP为 192.168.1.1): `curl -H Host:hello.test.com 192.168.1.1:23456`，也能够看到欢迎页面 `Welcome to nginx!`，说明ingress测试成功 + +#### 为 traefik WEB 管理页面创建 ingress 规则 + +`kubectl create -f /etc/ansible/manifests/ingress/traefik/traefik-ui.ing.yaml` + +``` bash +# traefik-ui.ing.yaml内容 +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: traefik-web-ui + namespace: kube-system +spec: + rules: + - host: traefik-ui.test.com + http: + paths: + - path: / + backend: + serviceName: traefik-ingress-service + servicePort: 8080 +``` + ++ 在集群外部可以使用 `curl -H Host:traefik-ui.test.com 192.168.1.1:23456` 尝试访问WEB管理页面，返回 `Found.`说明 traefik-ui的ingress配置生效了。 + ++ 在客户端主机也可以通过修改本机 `hosts` 文件，如上例子，增加两条记录： + +``` text +192.168.1.1 hello.test.com +192.168.1.1 traefik-ui.test.com +``` +打开浏览器输入域名 `http://hello.test.com:23456` 和 `http://traefik-ui.test.com:23456` 就可以访问k8s的应用服务了。 + +### 可选1: 使用`LoadBalancer`服务类型来暴露ingress，自有环境（非公有云）可以参考[metallb文档](metallb.md) + +``` bash +# 修改traefik-ingress 使用 LoadBalancer服务 +$ sed -i 's/NodePort$/LoadBalancer/g' /etc/ansible/manifests/ingress/traefik/traefik-ingress.yaml +# 创建traefik-ingress +$ kubectl apply -f /etc/ansible/manifests/ingress/traefik/traefik-ingress.yaml +# 验证 +$ kubectl get svc --all-namespaces |grep traefik +kube-system traefik-ingress-service LoadBalancer 10.68.163.243 192.168.1.241 80:23456/TCP,8080:37088/TCP 1m +``` +这时可以修改客户端本机 `hosts`文件：(如上例192.168.1.241) + +``` text +192.168.1.241 hello.test.com +192.168.1.241 traefik-ui.test.com +``` +打开浏览器输入域名 `http://hello.test.com` 和 `http://traefik-ui.test.com`可以正常访问。 + +### 可选2: 部署`ingress-service`的负载均衡 + +- 利用 nginx/haproxy 等集群，可以做代理转发以去掉 `23456`这个端口。如果你的集群根据本项目部署了高可用方案，那么可以利用`LB` 节点haproxy 来做，当然如果生产环境K8S应用已经部署非常多，建议还是使用独立的 `nginx/haproxy`集群。 + +具体参考[配置转发 ingress nodePort](../op/loadballance_ingress_nodeport.md)，如上配置访问集群`MASTER_IP`的`80`端口时，由haproxy代理转发到实际的node节点暴露的nodePort端口上了。这时可以修改客户端本机 `hosts`文件如下：(假定 MASTER_IP=192.168.1.10) + +``` text +192.168.1.10 hello.test.com +192.168.1.10 traefik-ui.test.com +``` +打开浏览器输入域名 `http://hello.test.com` 和 `http://traefik-ui.test.com`可以正常访问。 + +## 下一步[配置https ingress](ingress-tls.md) diff --git a/docs/guide/ipvs.md b/docs/guide/ipvs.md new file mode 100644 index 0000000..882a24d --- /dev/null +++ b/docs/guide/ipvs.md @@ -0,0 +1,35 @@ +# IPVS 服务负载均衡 + +kube-proxy 组件监听 API server 中 service 和 endpoint 的变化情况，从而为 k8s 集群内部的 service 提供动态负载均衡。在v1.10之前主要通过 iptables来实现，是稳定、推荐的方式，但是当服务多的时候会产生太多的 iptables 规则，大规模情况下有明显的性能问题；在v1.11 GA的 ipvs高性能负载模式，采用增量式更新，并可以保证 service 更新期间连接的保持。 + +- NOTE: k8s v1.11.0 CentOS7下使用ipvs模式会有问题（见 kubernetes/kubernetes#65461），测试 k8s v1.10.2 CentOS7 可以。 + +## 启用 ipvs + +k8s v1.11 版本启用 ipvs 十分方便，只要在 kube-proxy 启动参数（或者配置文件中）中增加 `--proxy-mode=ipvs`: + +``` bash +[Unit] +Description=Kubernetes Kube-Proxy Server +After=network.target + +[Service] +WorkingDirectory=/var/lib/kube-proxy +ExecStart={{ bin_dir }}/kube-proxy \ + --bind-address={{ NODE_IP }} \ + --hostname-override={{ NODE_IP }} \ + --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \ + --logtostderr=true \ + --proxy-mode=ipvs +Restart=on-failure +RestartSec=5 +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` + +目前 kubeasz 0.2.2 支持两种方式集成使用 ipvs：一种是使用kube-proxy 自带的ipvs 实现；另一种是使用 kube-router网络插件带的ipvs service-proxy； + +- v1.11, v1.10 版本启用 kube-proxy的 ipvs 模式：只需要在 `roles/kube-node/defaults/main.yml` 配置 `PROXY_MODE: "ipvs"` +- 启用 kube-router 的 ipvs service-proxy：需在 ansible hosts 配置选择 `CLUSTER_NETWORK="kube-router"`；另外在 `roles/kube-router/defaults/main.yml` 配置 `SERVICE_PROXY: "true"` diff --git a/docs/guide/istio.md b/docs/guide/istio.md new file mode 100644 index 0000000..f97c8f0 --- /dev/null +++ b/docs/guide/istio.md @@ -0,0 +1,173 @@ +--- +title: "Istio 1.0.3 安装　" +date: 2018-11-12T13:44:34+08:00 +draft: false +--- + +#### Service Mesh(服务网格) + +--- +Kubernetes 已经给我们带来了诸多的好处。但是仍有些需求比如 A/B 测试、金丝雀发布、限流、访问控制,端到端认证等需要运维人员进一步去解决。 + +Istio 是完全开源的服务网格,提供了一套完整的解决方案，可以透明地分层到现有的分布式应用程序上。对开发人员几乎无感的同时获得超能力。 + +如果想要现有的服务支持 Istio，只需要在当前的环境中部署一个特殊的 sidecar 代理，即可。 + +##### 前提 + +---- + +- 安装 Kubernetes 集群 1.9+ +- [安装 Helm](./helm.md) + +##### 准备 + +---- + +进入 [Istio release](https://github.com/istio/istio/releases) 页面下载最新版安装包(1.0.3)并解压到当前目录, + +``` +curl -L https://git.io/getLatestIstio | sh - + +ll istio-1.0.3/ +total 28 +drwxr-xr-x 2 root root 22 10月 26 07:36 bin +drwxr-xr-x 6 root root 79 10月 26 07:36 install +-rw-r--r-- 1 root root 648 10月 26 07:36 istio.VERSION +-rw-r--r-- 1 root root 11343 10月 26 07:36 LICENSE +-rw-r--r-- 1 root root 5817 10月 26 07:36 README.md +drwxr-xr-x 12 root root 212 10月 26 07:36 samples +drwxr-xr-x 8 root root 4096 10月 26 07:36 tools +``` +- install Kubernetes 安装所需的 .yaml 文件 +- samples Task中的示例应用 +- bin/istioctl 客户端工具 +- istio.VERSION 配置文件 + +#### 安装 + +--- + +##### 安装　istio +注意事项 + +Istio 默认使用‘负载均衡器’服务对象类型。对于裸机安装没有负载均衡器的情况下，安装需指定‘NodePort’类型。 + +``` +helm install --name istio install/kubernetes/helm/istio --namespace istio-system --set gateways.istio-ingressgateway.type=NodePort --set gateways.istio-egressgateway.type=NodePort +``` + +##### 验证 +``` +kubectl get pod -n istio-system +NAME READY STATUS RESTARTS AGE +istio-citadel-6955bc9cb7-qh846 1/1 Running 0 3d22h +istio-egressgateway-7dc5cbbc56-k4cgh 1/1 Running 0 3d22h +istio-galley-545b6b8f5b-k7ssx 1/1 Running 0 3d22h +istio-ingressgateway-7958d776b5-ptdsc 1/1 Running 0 3d22h +istio-pilot-56bfdbffff-mtcn6 2/2 Running 0 3d22h +istio-policy-5c689f446f-6bzlq 2/2 Running 0 3d15h +istio-policy-5c689f446f-dvmfq 2/2 Running 0 3d22h +istio-policy-5c689f446f-f2kl8 2/2 Running 0 3d3h +istio-policy-5c689f446f-nfv2l 2/2 Running 0 3d1h +istio-policy-5c689f446f-qdtql 2/2 Running 0 3d2h +istio-sidecar-injector-99b476b7b-dt24k 1/1 Running 0 3d22h +istio-telemetry-55d68b5dfb-52ftl 2/2 Running 0 3d22h +istio-telemetry-55d68b5dfb-dvdvz 2/2 Running 0 3d22h +istio-telemetry-55d68b5dfb-ln2sr 2/2 Running 0 3d +istio-telemetry-55d68b5dfb-m2mb8 2/2 Running 0 3d +istio-telemetry-55d68b5dfb-sjgq8 2/2 Running 0 3d +prometheus-65d6f6b6c-dsv26 1/1 Running 0 3d22h + +``` +``` +kubectl get svc -n istio-system +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +istio-citadel ClusterIP 10.68.7.100 8060/TCP,9093/TCP 3d22h +istio-egressgateway NodePort 10.68.67.237 80:30060/TCP,443:38194/TCP 3d22h +istio-galley ClusterIP 10.68.12.54 443/TCP,9093/TCP 3d22h +istio-ingressgateway NodePort 10.68.87.79 80:31380/TCP,443:31390/TCP,31400:31400/TCP,15011:31812/TCP,8060:30957/TCP,853:23011/TCP,15030:22292/TCP,15031:23663/TCP 3d22h +istio-pilot ClusterIP 10.68.84.101 15010/TCP,15011/TCP,8080/TCP,9093/TCP 3d22h +istio-policy ClusterIP 10.68.94.206 9091/TCP,15004/TCP,9093/TCP 3d22h +istio-sidecar-injector ClusterIP 10.68.191.221 443/TCP 3d22h +istio-telemetry ClusterIP 10.68.199.8 9091/TCP,15004/TCP,9093/TCP,42422/TCP 3d22h +prometheus ClusterIP 10.68.91.13 9090/TCP +``` + +##### Sidecar 的自动注入 + +注意事项 + +需要在kube-apiserver 启动 admission-control 参数中加入 MutatingAdmissionWebhook 和 ValidatingAdmissionWebhook并确保正确的顺序,如果是多master安装，确保每个kube-apiserver都要进行修改。 + +``` +/bin/kube-apiserver --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook +``` + +重启 kube-apiserver 即可 + +##### 部署应用验证 + +istio 的samples目录中有很多示例。我们现在使用samples/sleep/sleep.yaml 来验证刚刚开启的Sidecar自动注入功能。 + +进入目录 istio-1.0.3/ 部署一个新的应用 + +``` +cd istio-1.0.3/ +kubectl apply -f samples/sleep/sleep.yaml + +kubectl get pod +NAME READY STATUS RESTARTS AGE +sleep-7549f66447-wv8cl 1/1 Running 0 1m +``` + +一切都是熟悉的味道。下面给 default 命名空间设置标签：istio-injection=enabled，这样就会在pod 创建时触发 Sidecar 的注入过程。从此default 名称空间拥有了超能力. + +``` +kubectl label namespace default istio-injection=enabled +kubectl get namespace -L istio-injection +NAME STATUS AGE ISTIO-INJECTION +default Active 1h enabled +istio-system Active 3d22h +kube-public Active 4d2h +kube-system Active 4d2h +``` +接下来删除上面创建的pod，观察下有什么变化。 + +``` +kubectl delete pod sleep-7549f66447-wv8cl +pod "sleep-7549f66447-wv8cl" deleted + +kubectl get pod +NAME READY STATUS RESTARTS AGE +sleep-7549f66447-x4td6 2/2 Running 0 37s +``` +刚刚的pod里面现在已经拥有两个容器，进入pod一探究竟。 +``` + kubectl describe pod sleep-7549f66447-x4td6 + + .... + + Containers: + sleep: + + .... + + istio-proxy: + + .... + +``` +多出了一个 istio-proxy 容器及其对应的存储卷 + + +#### 卸载istio + +--- + +``` +helm delete --purge istio + +``` + + diff --git a/docs/guide/jenkins.md b/docs/guide/jenkins.md new file mode 100644 index 0000000..800395b --- /dev/null +++ b/docs/guide/jenkins.md @@ -0,0 +1,180 @@ +# Jenkins CI/CD + +## 前言 +本文档介绍如何快速通过K8s集群实现Jenkins 动态Slave CI/CD流程。 + +## 开始之前 +在开始之前需要准备以下环境： +- k8s dns组件 +参考文档：[kubedns](kubedns.md) +- helm +为了简化部署，通过helm来安装Jenkins，可参考文档：[helm](helm.md) +- 持久化存储 +这里使用**NFS**演示，参考文档：[cluster-storage](../setup/08-cluster-storage.md)。 +如果k8s集群是部署在公有云，也可使用厂商的NAS等存储方案，项目中已集成支持阿里云NAS，其他的方案参考相关厂商文档 + +- Ingress Controller(nginx-ingress/traefik) +默认是通过Ingress访问Jenkins，因此需要安装一种`Ingress Controller`。参考文档：[ingress](ingress.md) +- Gitlab 代码管理仓库 +用于提交代码后自动触发CI, 目前项目中还没有相关内容，可[参考官网](https://about.gitlab.com/installation/)进行安装。 + +## 安装Jenkins +执行以下命令快速安装： +``` +helm install manifests/jenkins/ --name jenkins +``` +如果通过/etc/ansible/roles/helm/helm.yml安装的helm，安装过程会出现如下错误 + +``` bash +E0703 08:40:22.376225 19888 portforward.go:331] an error occurred forwarding 41655 -> 44134: error forwarding port 44134 to pod 5098414beaaa07140a4ba3240690b1ce989ece01e5db33db65eec83bd64bdedf, uid : exit status 1: 2018/07/03 08:40:22 socat[19991] E write(5, 0x1aec120, 3424): Connection reset by peer +Error: transport is closing +``` +请执行以下命令快速安装进行修复： +``` +helm install --tls manifests/jenkins/ --name jenkins +``` + +由于初始化过程中，默认安装指定的插件，所以启动较慢，大概5-10分钟左右就可以启动完成了。 + +部分默认配置说明： +**注**：以下配置都定义在`manifests/jenkins/values.yaml`文件中。 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
字段说明默认值
InstallPlugins初始化安装的插件 +
    +
  • kubernetes:1.6.3
    • +
  • workflow-aggregator:2.5
    • +
  • workflow-job:2.21
    • +
  • credentials-binding:1.16
    • +
  • git:3.9.0
    • +
  • gitlab:1.5.6
    • +
+
HostNameIngress访问入口jenkins.local.com
AdminPasswordadmin登录密码admin
UpdateCenter插件下载镜像地址https://mirrors.tuna.tsinghua.edu.cn/jenkins
StorageClass持久化存储SCnfs-dynamic-class
+ + +## 配置Kubernetes plugin +登录Jenkins，点击左边导航`系统管理`——>`系统设置`，拖动到最下面可以看到`云——>Kubernetes`配置，默认配置有以下字段： + +- Name：配置名称，后面运行测试的时候会用到，用于区别多个Kubernetes配置，默认为：kubernetes +- Kubernetes URL：集群访问url，可通过`kubectl cluster-info`查看，如果集群有部署**DNS**插件, 也可以直接填服务名称(自动解析)，默认使用服务名称：https://kubernetes +- Jenkins URL：Jenkins访问地址，默认使用服务名称+端口号 + +在Jenkins初始化时，默认都已经配置好了，可以直接新建项目测试了。 + +## 简单测试 +点击左边：新建任务——>流水线(Pipeline) +任务名称可以随便起，这里为：k8s-test +配置——>流水线，选择`Pipeline script` +以下为测试脚本内容： +``` +podTemplate(label: 'jenkins-slave', cloud: 'kubernetes') +{ + node ('jenkins-slave') { + stage('test') { + echo "hello, world" + sleep 60 + } + } +} +``` + +- cloud：插件配置中的Name +- label：插件配置中的Images——>Kubernetes Pod Tempalte——>Labels +- node：与label一致即可 + +保存配置，点击立即构建，查看控制台输出，出现以下内容就表示运行成功了： +``` +Agent default-lsths is provisioned from template Kubernetes Pod Template +Agent specification [Kubernetes Pod Template] (jenkins-slave): +* [jnlp] jenkins/jnlp-slave:alpine(resourceRequestCpu: 200m, resourceRequestMemory: 256Mi, resourceLimitCpu: 200m, resourceLimitMemory: 256Mi) + +Running on default-lsths in /home/jenkins/workspace/k8s-test +[Pipeline] { +[Pipeline] stage +[Pipeline] { (test) +[Pipeline] echo +hello, world +[Pipeline] sleep +Sleeping for 1 min 0 sec +[Pipeline] } +[Pipeline] // stage +[Pipeline] } +[Pipeline] // node +[Pipeline] } +[Pipeline] // podTemplate +[Pipeline] End of Pipeline +Finished: SUCCESS +``` + + +## 配置自动触发CI + +- 配置Gitlab项目 +在`Gitlab`中创建一个测试项目，将上面测试的脚本内容写入到一个`Jenkinsfile`文件中，然后上传到该测试项目根路径下。 + +- 配置Jenkins项目 +点击项目`配置`——>`构建触发器`——>勾选`Build when a change is pushed to GitLab. GitLab webhook URL:http://jenkins.local.com/project/k8s-test`——>保存配置 + +- 配置Webhook +进入Gitlab测试项目的`Settings——>Integrations`，一般只需要填写`URL`即可，其他的可根据需求环境配置 +默认Jenkins配置不允许匿名用户触发构建，因此还需要添加用户和token。 +URL的格式为： +`http://[UserID]:[API Token]@jenkins.local.com/project/[ProjectName]` + +Jenkins 用户ID Token查看： +点击右上角的`用户名——>设置——>API Token(点击Show API Token...)` + +最终Webhook中的URL类似： +http://admin:a910b1492e39e9dd1ea48ea7f7638aaf@jenkins.local.com/project/k8s-test + +后面只需要我们一提交代码到Git仓库，就会自动触发Jenkins进行构建了。 + +## 项目应用 +这里我们以一个简单的Java项目为例，实战演示如何进行CI/CD。 +基本环境配置上面已经说过了，这里就不多介绍。 +示例项目：https://github.com/lusyoe/springboot-k8s-example + +结构说明： +- 镜像构建文件：`Dockerfile` +- k8s应用配置：`k8s-example.yaml` +- 项目源码：`src` +- Jenkins构建文件：`jenkins/Jenkinsfile` + +构建流程说明： +- 通过Jenkins kubernetes插件，定义构建过程中所需的3个docker容器：maven、docker、kubectl (这3个容器都在一个pod中) +- 挂载docker.sock和kubeconfig文件 +- 首先使用`maven`容器，检出代码，执行项目构建 +- 使用`docker`容器，构建镜像，推送到镜像参考 +- 使用`kubectl`容器，部署`k8s-example`应用(这里后面也可以使用helm) + +访问： +项目通过Ingress访问`k8s-example.com`，出现`hello, world`,就表示服务部署成功了。 diff --git a/docs/guide/kernel_upgrade.md b/docs/guide/kernel_upgrade.md new file mode 100644 index 0000000..6409530 --- /dev/null +++ b/docs/guide/kernel_upgrade.md @@ -0,0 +1,45 @@ +# Linux Kernel 升级 + +k8s,docker,cilium等很多功能、特性需要较新的linux内核支持，所以有必要在集群部署前对内核进行升级；CentOS7 和 Ubuntu16.04可以很方便的完成内核升级。 + +## CentOS7 + +``` bash +# 载入公钥 +rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org +# 安装ELRepo +rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm +# 载入elrepo-kernel元数据 +yum --disablerepo=\* --enablerepo=elrepo-kernel repolist +# 查看可用的rpm包 +yum --disablerepo=\* --enablerepo=elrepo-kernel list kernel* +# 安装最新版本的kernel +yum --disablerepo=\* --enablerepo=elrepo-kernel install -y kernel-ml.x86_64 +# 删除旧版本工具包 +yum remove kernel-tools-libs.x86_64 kernel-tools.x86_64 -y +# 安装新版本工具包 +yum --disablerepo=\* --enablerepo=elrepo-kernel install -y kernel-ml-tools.x86_64 + +#查看默认启动顺序 +awk -F\' '$1=="menuentry " {print $2}' /etc/grub2.cfg +CentOS Linux (4.4.4-1.el7.elrepo.x86_64) 7 (Core) +CentOS Linux (3.10.0-327.10.1.el7.x86_64) 7 (Core) +CentOS Linux (0-rescue-c52097a1078c403da03b8eddeac5080b) 7 (Core) +#默认启动的顺序是从0开始，新内核是从头插入（目前位置在0，而4.4.4的是在1），所以需要选择0。 +grub2-set-default 0 +#重启并检查 +reboot +``` + +## Ubuntu16.04 + +``` bash +打开 http://kernel.ubuntu.com/~kernel-ppa/mainline/ 并选择列表中选择你需要的版本（以4.16.3为例）。 +接下来，根据你的系统架构下载 如下.deb 文件： +Build for amd64 succeeded (see BUILD.LOG.amd64): + linux-headers-4.16.3-041603_4.16.3-041603.201804190730_all.deb + linux-headers-4.16.3-041603-generic_4.16.3-041603.201804190730_amd64.deb + linux-image-4.16.3-041603-generic_4.16.3-041603.201804190730_amd64.deb +#安装后重启即可 +$ sudo dpkg -i *.deb +``` diff --git a/docs/guide/kubedns.md b/docs/guide/kubedns.md new file mode 100644 index 0000000..67c74ee --- /dev/null +++ b/docs/guide/kubedns.md @@ -0,0 +1,84 @@ +## 部署集群 DNS + +DNS 是 k8s 集群首先需要部署的，集群中的其他 pods 使用它提供域名解析服务；主要可以解析 `集群服务名 SVC` 和 `Pod hostname`；目前 k8s v1.9+ 版本可以有两个选择：`kube-dns` 和 `coredns`，可以选择其中一个部署安装。 + +### 部署 dns + +配置文件参考 `https://github.com/kubernetes/kubernetes` 项目目录 `kubernetes/cluster/addons/dns` + ++ 安装 + +``` bash +# 安装 kube-dns +$ kubectl create -f /etc/ansible/manifests/kubedns + +# 或者选择安装 coredns +$ kubectl create -f /etc/ansible/manifests/coredns +``` + ++ 集群 pod默认继承 node的dns 解析，修改 kubelet服务启动参数 --resolv-conf=""，可以更改这个特性，详见 kubelet 启动参数 + +### 验证 dns服务 + +新建一个测试nginx服务 + +`kubectl run nginx --image=nginx --expose --port=80` + +确认nginx服务 + +``` bash +kubectl get pod|grep nginx +nginx-7cbc4b4d9c-fl46v 1/1 Running 0 1m +kubectl get svc|grep nginx +nginx ClusterIP 10.68.33.167 80/TCP 1m +``` + +测试pod alpine + +``` bash +kubectl run test --rm -it --image=alpine /bin/sh +If you don't see a command prompt, try pressing enter. +/ # cat /etc/resolv.conf +nameserver 10.68.0.2 +search default.svc.cluster.local. svc.cluster.local. cluster.local. +options ndots:5 +# 测试集群内部服务解析 +/ # nslookup nginx.default.svc.cluster.local +Server: 10.68.0.2 +Address 1: 10.68.0.2 kube-dns.kube-system.svc.cluster.local + +Name: nginx +Address 1: 10.68.33.167 nginx.default.svc.cluster.local +/ # nslookup kubernetes.default.svc.cluster.local +Server: 10.68.0.2 +Address 1: 10.68.0.2 kube-dns.kube-system.svc.cluster.local + +Name: kubernetes +Address 1: 10.68.0.1 kubernetes.default.svc.cluster.local +# 测试外部域名的解析，默认集成node的dns解析 +/ # nslookup www.baidu.com +Server: 10.68.0.2 +Address 1: 10.68.0.2 kube-dns.kube-system.svc.cluster.local + +Name: www.baidu.com +Address 1: 180.97.33.108 +Address 2: 180.97.33.107 +/ # +``` + +- Note1: 如果你使用`calico`网络组件，通过命令`ansible-playbook 90.setup.yml`安装完集群后，直接安装dns组件，可能会出现如下BUG，分析是因为calico分配pod地址时候会从网段的第一个地址（网络地址）开始，详见提交的 [ISSUE #1710](https://github.com/projectcalico/calico/issues/1710)，临时解决办法为手动删除POD，重新创建后获取后面的IP地址 + +``` +# BUG出现现象 +$ kubectl get pod --all-namespaces -o wide +NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE +default busy-5cc98488d4-s894w 1/1 Running 0 28m 172.20.24.193 192.168.97.24 +kube-system calico-kube-controllers-6597d9c664-nq9hn 1/1 Running 0 1h 192.168.97.24 192.168.97.24 +kube-system calico-node-f8gnf 2/2 Running 0 1h 192.168.97.24 192.168.97.24 +kube-system kube-dns-69bf9d5cc9-c68mw 0/3 CrashLoopBackOff 27 31m 172.20.24.192 192.168.97.24 + +# 解决办法，删除pod，自动重建 +$ kubectl delete pod -n kube-system kube-dns-69bf9d5cc9-c68mw +``` + +- Note2: 使用``` kubectl run test -it --rm --image=busybox /bin/sh``` 进行解析测试可能会失败, busybox内的nslookup程序有bug, 详见 https://github.com/kubernetes/dns/issues/109 diff --git a/docs/guide/metallb.md b/docs/guide/metallb.md new file mode 100644 index 0000000..989e116 --- /dev/null +++ b/docs/guide/metallb.md @@ -0,0 +1,110 @@ +# metallb 网络负载均衡 + +`Metallb`是在自有硬件上（非公有云）实现 `Kubernetes Load-balancer`的工具，由`google`团队开源，值得推荐！项目[github主页](https://github.com/google/metallb)。 + +## metallb 简介 + +这里简单介绍下它的实现原理，具体可以参考[metallb官网](https://metallb.universe.tf/)，文档非常简洁、清晰。目前有如下的使用限制： + +- `Kubernetes v1.9.0`版本以上，暂不支持`ipvs`模式 +- 支持网络组件 (flannel/weave/romana), calico 部分支持 +- `layer2`和`bgp`两种模式，其中`bgp`模式需要外部网络设备支持`bgp`协议 + +`metallb`主要实现了两个功能：地址分配和对外宣告 + +- 地址分配：需要向网络管理员申请一段ip地址，如果是layer2模式需要这段地址与node节点地址同个网段（同一个二层）；如果是bgp模式没有这个限制。 +- 对外宣告：layer2模式使用arp协议，利用节点的mac额外宣告一个loadbalancer的ip（同mac多ip）；bgp模式下节点利用bgp协议与外部网络设备建立邻居，宣告loadbalancer的地址段给外部网络。 + +## kubeasz 集成安装metallb + +因bgp模式需要外部路由器的支持，这里主要选用layer2模式（如需选择bgp模式，相应修改roles/cluster-addon/templates/metallb/bgp.yaml.j2）。 + +- 1.修改roles/cluster-addon/defaults/main.yml 配置文件相关 + +``` bash +# metallb 自动安装 +metallb_install: "yes" +# 模式选择: 二层 "layer2" 或者三层 "bgp" +metallb_protocol: "layer2" +metallb_offline: "metallb_v0.7.3.tar" +metallb_vip_pool: "192.168.1.240/29" # 选一段与node节点相同网段的地址 +``` + +- 2.执行安装 `ansible-playbook 07.cluster-addon.yml`，其中controller 负责统一loadbalancer地址管理和服务监控，speaker 负责节点的loadbalancer地址的对外宣告（使用arp或者bgp网络协议），注意 **speaker是以DaemonSet 形式运行且只会调度到有node-role.kubernetes.io/metallb-speaker=true标签的节点**，所以你可以选择做speaker的节点（该节点网络性能要好），使用命令 `$ kubectl label nodes 192.168.1.43 node-role.kubernetes.io/metallb-speaker=true` + +- 3.验证metallb相关 pod + +``` bash +$ kubectl get node +NAME STATUS ROLES AGE VERSION +192.168.1.41 Ready,SchedulingDisabled master 4h v1.11.3 +192.168.1.42 Ready node 4h v1.11.3 +192.168.1.43 Ready metallb-speaker,node 4h v1.11.3 +192.168.1.44 Ready metallb-speaker,node 4h v1.11.3 +$ kubectl get pod -n metallb-system +NAME READY STATUS RESTARTS AGE +controller-9c57dbd4-798nb 1/1 Running 0 4h +speaker-9rjmk 1/1 Running 0 4h +speaker-n79l4 1/1 Running 0 4h +``` + +- 3.创建测试应用验证 loadbalancer 地址分配 + +``` bash +# 创建测试应用 +$ cat > test-nginx.yaml << EOF +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: nginx3 +spec: + selector: + matchLabels: + app: nginx3 + template: + metadata: + labels: + app: nginx3 + spec: + containers: + - name: nginx3 + image: nginx:1 + ports: + - name: http + containerPort: 80 + +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx3 +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 80 + selector: + app: nginx3 + type: LoadBalancer +EOF +$ kubectl apply -f test-nginx.yaml + +# 查看生成的loadbalancer 地址，如下验证成功 +$ kubectl get svc +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +kubernetes ClusterIP 10.68.0.1 443/TCP 5h +nginx3 LoadBalancer 10.68.82.227 192.168.1.240 80:38702/TCP 1m +``` + +- 4.验证使用loadbalacer 来暴露ingress的服务地址，之前在[ingress文档](ingress.md)中我们是使用nodeport方式服务类型，现在我们可以方便的使用loadbalancer类型了，使用loadbalancer地址(192.168.1.241)方便的绑定你要的域名进行访问。 + +``` bash +# 修改traefik-ingress 使用 LoadBalancer服务 +$ sed -i 's/NodePort$/LoadBalancer/g' /etc/ansible/manifests/ingress/traefik/traefik-ingress.yaml +# 创建traefik-ingress +$ kubectl apply -f /etc/ansible/manifests/ingress/traefik/traefik-ingress.yaml +# 验证 +$ kubectl get svc --all-namespaces |grep traefik +kube-system traefik-ingress-service LoadBalancer 10.68.163.243 192.168.1.241 80:23456/TCP,8080:37088/TCP 1m +``` diff --git a/docs/guide/metrics-server.md b/docs/guide/metrics-server.md new file mode 100644 index 0000000..e964d5e --- /dev/null +++ b/docs/guide/metrics-server.md @@ -0,0 +1,60 @@ +# Metrics Server + +从 v1.8 开始，资源使用情况的度量（如容器的 CPU 和内存使用）可以通过 Metrics API 获取；前提是集群中要部署 Metrics Server，它从Kubelet 公开的Summary API采集指标信息，关于更多的背景介绍请参考如下文档： +- Metrics Server[设计提案](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/metrics-server.md) + +大致是说它符合k8s的监控架构设计，受heapster项目启发，并且比heapster优势在于：访问不需要apiserver的代理机制，提供认证和授权等；很多集群内组件依赖它（HPA,scheduler,kubectl top），因此它应该在集群中默认运行；部分k8s集群的安装工具已经默认集成了Metrics Server的安装，以下概述下它的安装： + +- 1.metric-server是扩展的apiserver，依赖于[kube-aggregator](https://github.com/kubernetes/kube-aggregator)，因此需要在apiserver中开启相关参数。 +- 2.需要在集群中运行deployment处理请求 + +从kubeasz 0.1.0 开始，metrics-server已经默认集成在集群安装脚本中，请查看`roles/cluster-addon/defaults/main.yml`中的设置 + +## 安装 + +默认已集成在90.setup.yml中，如果分步请执行`ansible-play /etc/ansible/07.cluster-addon.yml` + +- 1.设置apiserver相关[参数](../../roles/kube-master/templates/kube-apiserver.service.j2) +``` bash +... # 省略 + --requestheader-client-ca-file={{ ca_dir }}/ca.pem \ + --requestheader-allowed-names=aggregator \ + --requestheader-extra-headers-prefix=X-Remote-Extra- \ + --requestheader-group-headers=X-Remote-Group \ + --requestheader-username-headers=X-Remote-User \ + --proxy-client-cert-file={{ ca_dir }}/aggregator-proxy.pem \ + --proxy-client-key-file={{ ca_dir }}/aggregator-proxy-key.pem \ + --enable-aggregator-routing=true \ +``` +- 2.生成[aggregator proxy相关证书](../../roles/kube-master/tasks/main.yml) + +参考1：https://kubernetes.io/docs/tasks/access-kubernetes-api/configure-aggregation-layer/ +参考2：https://kubernetes.io/docs/tasks/access-kubernetes-api/setup-extension-api-server/ + +## 验证 + +- 查看生成的新api：v1beta1.metrics.k8s.io +``` bash +$ kubectl get apiservice|grep metrics +v1beta1.metrics.k8s.io 1d +``` + +- 查看kubectl top命令（无需额外安装heapster） +``` bash +$ kubectl top node +NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% +192.168.1.1 116m 2% 2342Mi 60% +192.168.1.2 79m 1% 1824Mi 47% +192.168.1.3 82m 2% 1897Mi 49% +$ kubectl top pod --all-namespaces # 输出略 +``` + +- 验证基于metrics-server实现的基础hpa自动缩放，请参考[hpa.md](hpa.md) + +## 补充 + +目前dashboard插件如果想在界面上显示资源使用率，它还依赖于`heapster`；另外，测试发现k8s 1.8版本的`kubectl top`也依赖`heapster`，因此建议补充安装`heapster`，无需安装`influxdb`和`grafana`。 + +``` bash +$ kubectl apply -f /etc/ansible/manifests/heapster/heapster.yaml +``` diff --git a/docs/guide/networkpolicy.md b/docs/guide/networkpolicy.md new file mode 100644 index 0000000..a08635d --- /dev/null +++ b/docs/guide/networkpolicy.md @@ -0,0 +1,189 @@ +## Network Policy + +`Network Policy`提供了基于策略的网络控制，用于隔离应用并减少攻击面。它使用标签选择器模拟传统的分段网络，并通过策略控制它们之间的流量以及来自外部的流量；目前基于`linux iptables`实现，使用类似`nf_conntrack`检查记录网络流量`session`从而决定流量是否阻断；因此它是`状态检测防火墙`。 + +- `k8s v1.7`版本以后，Network Policy已经默认可用；`v1.6`以及之前版本需要在kube-apiserver中开启extensions/v1beta1/networkpolicies +- 网络插件要支持 Network Policy，如 Calico、Romana、Weave Net + +### 简单示例 + +实验环境：k8s v1.9, calico 2.6.5 + +首先部署测试用nginx服务 + +``` bash +$ kubectl run nginx --image=nginx --replicas=3 --port=80 --expose +# 验证测试nginx服务 +$ kubectl get pod -o wide +NAME READY STATUS RESTARTS AGE IP NODE +nginx-7587c6fdb6-p2fpz 1/1 Running 0 55m 172.20.125.2 10.0.96.7 +nginx-7587c6fdb6-pbw7c 1/1 Running 0 55m 172.20.124.2 10.0.96.6 +nginx-7587c6fdb6-v48db 1/1 Running 0 55m 172.20.121.195 10.0.96.4 +$ kubectl get svc nginx +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +nginx ClusterIP 10.68.7.183 80/TCP 1h +``` +默认情况下，其他pod可以访问nginx服务 + +``` bash +$ kubectl run busy1 --rm -it --image=busybox /bin/sh +If you don't see a command prompt, try pressing enter. +/ # wget --spider --timeout=1 nginx +Connecting to nginx (10.68.7.183:80) +``` +创建`DefaultDeny Network Policy`后，其他Pod（包括namespace外部）不能访问nginx + +``` bash +$ cat > default-deny.yaml << EOF +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny +spec: + podSelector: {} + policyTypes: + - Ingress +EOF +$ kubectl create -f default-deny.yaml +networkpolicy "default-deny" created +$ kubectl run busy1 --rm -it --image=busybox /bin/sh +If you don't see a command prompt, try pressing enter. +/ # wget --spider --timeout=1 nginx +Connecting to nginx (10.68.7.183:80) +wget: download timed out +``` +创建一个允许带有access=true的Pod访问nginx的网络策略 + +``` bash +$ cat > nginx-policy.yaml << EOF +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: access-nginx +spec: + podSelector: + matchLabels: + run: nginx + ingress: + - from: + - podSelector: + matchLabels: + access: "true" +EOF +$ kubectl create -f nginx-policy.yaml +networkpolicy "access-nginx" created + +# 不带access=true标签的Pod还是无法访问nginx服务 +$ kubectl run busy1 --rm -it --image=busybox /bin/sh +If you don't see a command prompt, try pressing enter. +/ # wget --spider --timeout=1 nginx +Connecting to nginx (10.68.7.183:80) +wget: download timed out + +# 而带有access=true标签的Pod可以访问nginx服务 +$ kubectl run busy2 --rm -it --labels="access=true" --image=busybox /bin/sh +If you don't see a command prompt, try pressing enter. +/ # wget --spider --timeout=1 nginx +Connecting to nginx (10.68.7.183:80) +``` + +### 示例策略解读 + +``` bash +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: test-network-policy + namespace: default +spec: + podSelector: + matchLabels: + role: db + policyTypes: + - Ingress + - Egress + ingress: + - from: + - ipBlock: + cidr: 172.17.0.0/16 + except: + - 172.17.1.0/24 + - namespaceSelector: + matchLabels: + project: myproject + - podSelector: + matchLabels: + role: frontend + ports: + - protocol: TCP + port: 6379 + egress: + - to: + - ipBlock: + cidr: 10.0.0.0/24 + ports: + - protocol: TCP + port: 5978 +``` +- 策略作用的对象Pods：default命名空间下带有`role=db`标签的Pod + - 内向流量策略 + - 允许属于`172.17.0.0/16`网段但不属于`172.17.1.0/24`的源地址访问该对象Pods的TCP 6379端口 + - 允许带有project=myprojects标签的namespace中所有Pod访问该对象Pods的TCP 6379端口 + - 允许default命名空间下带有role=frontend标签的Pod访问该对象Pods的TCP 6379端口 + - 拒绝其他所有主动访问该对象Pods的网络流量 + - 外向流量策略 + - 允许该对象Pods主动访问目的地址属于`10.0.0.0/24`网段且目的端口为TCP 5978的流量 + - 拒绝该对象Pods其他所有主动外向网络流量 + +### 使用场景 + +参考阅读[ahmetb/kubernetes-network-policy-recipes](https://github.com/ahmetb/kubernetes-network-policy-recipes) 该项目举例一些使用NetworkPolicy的场景，并有形象的配图 + +#### 拒绝其他namespaces访问服务 + +![deny_from_other_namespaces](../../pics/deny_from_other_namespaces.gif) + ++ 场景1：你的k8s集群应用按照namespaces区分生产、测试环境，你要确保生产环境不会受到测试环境错误访问影响 ++ 场景2：你的k8s集群有多租户应用采用namespaces区分的，你要确保多租户之间的应用隔离 + +在你需要隔离的命名空间创建如下策略: + +``` bash +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + namespace: your-ns + name: deny-other-namespaces +spec: + podSelector: + matchLabels: + ingress: + - from: + - podSelector: {} +``` + +#### 允许外部访问服务 + ++ 场景：暴露特定Pod的特定端口给外部访问 + +![allow_from_external](../../pics/allow_from_external.gif) + +``` bash +# 创建示例应用待暴露服务 +$ kubectl run web --image=nginx --labels=app=web --port 80 --expose + +# 创建网络策略 +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: web-allow-external +spec: + podSelector: + matchLabels: + app: web + ingress: + - from: [] + ports: + - protocol: TCP + port: 80 +``` diff --git a/docs/guide/nfs-server.md b/docs/guide/nfs-server.md new file mode 100644 index 0000000..1efdc29 --- /dev/null +++ b/docs/guide/nfs-server.md @@ -0,0 +1,81 @@ +## 创建 NFS 服务器 + +NFS 允许系统将其目录和文件共享给网络上的其他系统。通过 NFS，用户和应用程序可以访问远程系统上的文件，就象它们是本地文件一样。 + +### 安装 +Ubuntu 16.04 键入以下命令安装 NFS 服务器： + +``` bash +apt install nfs-kernel-server +``` + +### 配置 +编辑`/etc/exports`文件添加需要共享目录，每个目录的设置独占一行，编写格式如下： + +`NFS共享目录路径 客户机IP或者名称(参数1,参数2,...,参数n)` + +例如： + +``` bash +/home *(ro,sync,insecure,no_root_squash) +/share 192.168.1.*(rw,sync,insecure,no_subtree_check,no_root_squash) +``` +| 参数 | 说明 | +| :- | :- | +| ro | 只读访问 | +| rw | 读写访问 | +| sync | 所有数据在请求时写入共享 | +| async | nfs在写入数据前可以响应请求 | +| secure | nfs通过1024以下的安全TCP/IP端口发送 | +| insecure | nfs通过1024以上的端口发送 | +| wdelay | 如果多个用户要写入nfs目录，则归组写入（默认） | +| no_wdelay | 如果多个用户要写入nfs目录，则立即写入，当使用async时，无需此设置 | +| hide | 在nfs共享目录中不共享其子目录 | +| no_hide | 共享nfs目录的子目录 | +| subtree_check | 如果共享/usr/bin之类的子目录时，强制nfs检查父目录的权限（默认） | +| no_subtree_check | 不检查父目录权限 | +| all_squash | 共享文件的UID和GID映射匿名用户anonymous，适合公用目录 | +| no_all_squash | 保留共享文件的UID和GID（默认） | +| root_squash | root用户的所有请求映射成如anonymous用户一样的权限（默认） | +| no_root_squash | root用户具有根目录的完全管理访问权限 | +| anonuid=xxx | 指定nfs服务器/etc/passwd文件中匿名用户的UID | +| anongid=xxx | 指定nfs服务器/etc/passwd文件中匿名用户的GID | + ++ 注1：尽量指定主机名或IP或IP段最小化授权可以访问NFS 挂载的资源的客户端 ++ 注2：经测试参数insecure必须要加，否则客户端挂载出错mount.nfs: access denied by server while mounting + +### 启动 + +配置完成后，您可以在终端提示符后运行以下命令来启动 NFS 服务器： + +``` bash +systemctl start nfs-kernel-server.service +``` + +### 客户端挂载 + +Ubuntu 16.04，首先需要安装 `nfs-common` 包 + +``` bash +apt install nfs-common +``` +CentOS 7, 需要安装 `nfs-utils` 包 + +``` bash +yum install nfs-utils +``` + +使用 mount 命令来挂载其他机器共享的 NFS 目录。可以在终端提示符后输入以下类似的命令： + +``` bash +mount example.hostname.com:/ubuntu /local/ubuntu +``` +挂载点 /local/ubuntu 目录必须已经存在。而且在 /local/ubuntu 目录中没有文件或子目录。 + +另一个挂载NFS 共享的方式就是在 /etc/fstab 文件中添加一行。该行必须指明 NFS 服务器的主机名、服务器输出的目录名以及挂载 NFS 共享的本机目录。 + +以下是在 /etc/fstab 中的常用语法： + +``` bash +example.hostname.com:/ubuntu /local/ubuntu nfs rsize=8192,wsize=8192,timeo=14,intr +``` diff --git a/docs/guide/prometheus.md b/docs/guide/prometheus.md new file mode 100644 index 0000000..ef8f783 --- /dev/null +++ b/docs/guide/prometheus.md @@ -0,0 +1,151 @@ +# Prometheus +随着`heapster`项目停止更新并慢慢被`metrics-server`取代，集群监控这项任务也将最终转移。`prometheus`的监控理念、数据结构设计其实相当精简，包括其非常灵活的查询语言；但是对于初学者来说，想要在k8s集群中实践搭建一套相对可用的部署却比较麻烦，由此还产生了不少专门的项目（如：[prometheus-operator](https://github.com/coreos/prometheus-operator)），本文介绍使用`helm chart`部署集群的prometheus监控。 +- `helm`已成为`CNCF`独立托管项目，预计会更加流行起来 + +## 前提 + +- 安装 helm：以本项目[安全安装helm](helm.md)为例 +- 安装 [kube-dns](kubedns.md) + +## 准备 + +安装目录概览 `ll /etc/ansible/manifests/prometheus` + +``` bash +drwx------ 3 root root 4096 Jun 3 22:42 grafana/ +-rw-r----- 1 root root 67875 Jun 4 22:47 grafana-dashboards.yaml +-rw-r----- 1 root root 690 Jun 4 09:34 grafana-settings.yaml +-rw-r----- 1 root root 1105 May 30 16:54 prom-alertrules.yaml +-rw-r----- 1 root root 474 Jun 5 10:04 prom-alertsmanager.yaml +drwx------ 3 root root 4096 Jun 2 21:39 prometheus/ +-rw-r----- 1 root root 294 May 30 18:09 prom-settings.yaml +``` +- 目录`prometheus/`和`grafana/`即官方的helm charts，可以使用`helm fetch --untar stable/prometheus` 和 `helm fetch --untar stable/grafana`下载，本安装不会修改任何官方charts里面的内容，这样方便以后跟踪charts版本的更新 +- `prom-settings.yaml`：个性化prometheus安装参数，比如禁用PV，禁用pushgateway，设置nodePort等 +- `prom-alertrules.yaml`：配置告警规则 +- `prom-alertsmanager.yaml`：配置告警邮箱设置等 +- `grafana-settings.yaml`：个性化grafana安装参数，比如用户名密码，datasources，dashboardProviders等 +- `grafana-dashboards.yaml`：预设置dashboard + +## 安装 + +``` bash +$ source ~/.bashrc +$ cd /etc/ansible/manifests/prometheus +# 安装 prometheus chart，如果你的helm安装没有启用tls证书，请忽略--tls参数 +$ helm install --tls \ + --name monitor \ + --namespace monitoring \ + -f prom-settings.yaml \ + -f prom-alertsmanager.yaml \ + -f prom-alertrules.yaml \ + prometheus +# 安装 grafana chart +$ helm install --tls \ + --name grafana \ + --namespace monitoring \ + -f grafana-settings.yaml \ + -f grafana-dashboards.yaml \ + grafana +``` + +## 验证安装 + +``` bash +# 查看相关pod和svc +$ kubectl get pod,svc -n monitoring +NAME READY STATUS RESTARTS AGE +grafana-54dc76d47d-2mk55 1/1 Running 0 1m +monitor-prometheus-alertmanager-6d9d9b5b96-w57bk 2/2 Running 0 2m +monitor-prometheus-kube-state-metrics-69f5d56f49-fh9z7 1/1 Running 0 2m +monitor-prometheus-node-exporter-55bwx 1/1 Running 0 2m +monitor-prometheus-node-exporter-k8sb2 1/1 Running 0 2m +monitor-prometheus-node-exporter-kxlr9 1/1 Running 0 2m +monitor-prometheus-node-exporter-r5dx8 1/1 Running 0 2m +monitor-prometheus-server-5ccfc77dff-8h9k6 2/2 Running 0 2m + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +grafana NodePort 10.68.74.242 80:39002/TCP 1m +monitor-prometheus-alertmanager NodePort 10.68.69.105 80:39001/TCP 2m +monitor-prometheus-kube-state-metrics ClusterIP None 80/TCP 2m +monitor-prometheus-node-exporter ClusterIP None 9100/TCP 2m +monitor-prometheus-server NodePort 10.68.248.94 80:39000/TCP 2m +``` + +- 访问prometheus的web界面：`http://$NodeIP:39000` +- 访问alertmanager的web界面：`http://$NodeIP:39001` +- 访问grafana的web界面：`http://$NodeIP:39002` (默认用户密码 admin:admin，可在web界面修改) + +## 管理操作 + +- 升级（修改配置）：修改配置请在`prom-settings.yaml` `prom-alertsmanager.yaml` 等文件中进行，保存后执行： +``` bash +# 修改prometheus +$ helm upgrade --tls monitor -f prom-settings.yaml -f prom-alertsmanager.yaml -f prom-alertrules.yaml prometheus +# 修改grafana +$ helm upgrade --tls grafana -f grafana-settings.yaml -f grafana-dashboards.yaml grafana +``` +- 回退：具体可以参考`helm help rollback`文档 +``` bash +$ helm rollback --tls monitor [REVISION] +``` +- 删除 +``` bash +$ helm del --tls monitor --purge +$ helm del --tls grafana --purge +``` + +## 验证告警 + +- 修改`prom-alertsmanager.yaml`文件中邮件告警为有效的配置内容，并使用 helm upgrade更新安装 +- 手动临时关闭 master 节点的 kubelet 服务，等待几分钟看是否有告警邮件发送 + +``` bash +# 在 master 节点运行 +$ systemctl stop kubelet +``` + +## [可选] 配置钉钉告警 + +- 创建钉钉群，获取群机器人 webhook 地址 + +使用钉钉创建群聊以后可以方便设置群机器人，【群设置】-【群机器人】-【添加】-【自定义】-【添加】，然后按提示操作即可，参考 https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.666d4a97eCG7XA&treeId=257&articleId=105735&docType=1 + +上述配置好群机器人，获得这个机器人对应的Webhook地址，记录下来，后续配置钉钉告警插件要用，格式如下 + +``` +https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxx +``` + +- 创建钉钉告警插件，参考 http://theo.im/blog/2017/10/16/release-prometheus-alertmanager-webhook-for-dingtalk/ + +``` bash +# 编辑修改文件中 access_token=xxxxxx 为上一步你获得的机器人认证 token +$ vi /etc/ansible/manifests/prometheus/dingtalk-webhook.yaml +# 运行插件 +$ kubectl apply -f /etc/ansible/manifests/prometheus/dingtalk-webhook.yaml +``` + +- 修改 alertsmanager 告警配置后，更新 helm prometheus 部署，成功后如上节测试告警发送 + +``` bash +# 修改 alertsmanager 告警配置 +$ cd /etc/ansible/manifests/prometheus +$ vi prom-alertsmanager.yaml +# 增加 receiver dingtalk，然后在 route 配置使用 receiver: dingtalk + receivers: + - name: dingtalk + webhook_configs: + - send_resolved: false + url: http://webhook-dingtalk.monitoring.svc.cluster.local:8060/dingtalk/webhook1/send +# ... +# 更新 helm prometheus 部署 +$ helm upgrade --tls monitor -f prom-settings.yaml -f prom-alertsmanager.yaml -f prom-alertrules.yaml prometheus +``` + +## 下一步 + +- 继续了解prometheus查询语言和配置文件 +- 继续了解prometheus告警规则，编写适合业务应用的告警规则 +- 继续了解grafana的dashboard编写，本项目参考了部分[feisky的模板](https://grafana.com/orgs/feisky/dashboards) +如果对以上部分有心得总结，欢迎分享贡献在项目中。 diff --git a/docs/guide/rollingupdateWithZeroDowntime.md b/docs/guide/rollingupdateWithZeroDowntime.md new file mode 100644 index 0000000..f2fc37f --- /dev/null +++ b/docs/guide/rollingupdateWithZeroDowntime.md @@ -0,0 +1,215 @@ +## 1、前言 +在当下微服务架构盛行的时代，用户希望应用程序时时刻刻都是可用，为了满足不断变化的新业务，需要不断升级更新应用程序，有时可能需要频繁的发布版本。实现"零停机"、“零感知”的持续集成(Continuous Integration)和持续交付/部署(Continuous Delivery)应用程序，一直都是软件升级换代不得不面对的一个难题和痛点，也是一种追求的理想方式，也是DevOps诞生的目的。 +## 2、滚动发布 +把一次完整的发布过程，合理地分成多个批次，每次发布一个批次，**成功后**，再发布下一个批次，最终完成所有批次的发布。在整个滚动过程期间，保证始终有可用的副本在运行，从而平滑的发布新版本，实现**零停机(without an outage)**、用户**零感知**，是一种非常主流的发布方式。由于其自动化程度比较高，通常需要复杂的发布工具支撑，而k8s可以完美的胜任这个任务。 +## 3、k8s滚动更新机制 +**k8s创建副本应用程序的最佳方法就是部署(Deployment)，部署自动创建副本集(ReplicaSet)，副本集可以精确地控制每次替换的Pod数量，从而可以很好的实现滚动更新**。具体来说，k8s每次使用一个新的副本控制器(replication controller)来替换已存在的副本控制器，从而始终使用一个新的Pod模板来替换旧的pod模板。 +>大致步骤如下： +>1. 创建一个新的replication controller。 +>2. 增加或减少pod副本数量，直到满足当前批次期望的数量。 +>3. 删除旧的replication controller。 + +## 4、演示 +>使用kubectl更新一个已部署的应用程序，并模拟回滚。为了方便分析，将应用程序的pod副本数量设置为10。 +``` bash +$ kubectl run busy --image=busybox:1.28.4 sleep 36000000 --replicas=10 +``` +### 4.1. 发布微服务 +- 当前服务状态查看 +``` bash +# 查看部署列表 +root@kube-aio:~# kubectl get deploy busy +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +busy 10 10 10 10 5m + +# 查看正在运行的pod +root@kube-aio:~# kubectl get pod | grep busy +busy-794c95f5d7-56b6w 1/1 Running 0 5m +busy-794c95f5d7-8ddjr 1/1 Running 0 5m +busy-794c95f5d7-8zm8r 1/1 Running 0 5m +busy-794c95f5d7-9hjhp 1/1 Running 0 5m +busy-794c95f5d7-df2r2 1/1 Running 0 5m +busy-794c95f5d7-fsn94 1/1 Running 0 5m +busy-794c95f5d7-k4w8r 1/1 Running 0 5m +busy-794c95f5d7-lsmgb 1/1 Running 0 5m +busy-794c95f5d7-rg8kw 1/1 Running 0 5m +busy-794c95f5d7-xpxxt 1/1 Running 0 5m + +# 通过pod描述，查看应用程序的当前映像版本 +root@kube-aio:~# kubectl describe pod busy-794c95f5d7-56b6w |grep Image + Image: busybox:1.28.4 + Image ID: docker-pullable://busybox@sha256:141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47 +``` +- 升级镜像版本到1.29 + - 为了更清晰看到更新过程，可另开一个窗口使用`$ watch kubectl get deployment busy`实时查看变化 +``` bash +$ kubectl set image deployments/busy busy=busybox:1.29 +``` +### 4.2. 验证发布 +``` bash +# 检查rollout状态 +root@kube-aio:~# kubectl rollout status deployments/busy +deployment "busy" successfully rolled out + +# 检查pod详情 +root@kube-aio:~# kubectl describe pod busy-665cdb7b-44jnt |grep Image + Image: busybox:1.29 + Image ID: docker-pullable://busybox@sha256:cb63aa0641a885f54de20f61d152187419e8f6b159ed11a251a09d115fdff9bd +``` +从上面可以看到，镜像已经升级到1.29版本 +### 4.3. 回滚发布 +``` bash +# 回滚发布 +root@kube-aio:~# kubectl rollout undo deployments/busy +deployment.apps "busy" + +# 回滚完成 +root@kube-aio:~# kubectl rollout status deployments/busy +deployment "busy" successfully rolled out + +# 镜像又回退到1.28.4 版本 +root@kube-aio:~# kubectl describe pod busy-794c95f5d7-4x9bn |grep Image + Image: busybox:1.28.4 + Image ID: docker-pullable://busybox@sha256:141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47 +``` + +到目前为止，整个滚动发布工作就圆满完成了！！！ +**那么如果我们想回滚到指定版本呢？答案是k8s完美支持，并且还可以通过资源文件进行配置保留的历史版次量**。由于篇幅有限，感兴趣的朋友，可以自己下去实战，回滚命令如下： +```javascript +kubectl rollout undo deployment/busy --to-revision=<版次> +``` +## 5、原理 +k8s精确地控制着整个发布过程，分批次有序地进行着滚动更新，直到把所有旧的副本全部更新到新版本。实际上，k8s是通过两个参数来精确地控制着每次滚动的pod数量： + +>* **`maxSurge` 滚动更新过程中运行操作期望副本数的最大pod数，可以为绝对数值(eg：5)，但不能为0；也可以为百分数(eg：10%)。** +>* **`maxUnavailable` 滚动更新过程中不可用的最大pod数，可以为绝对数值(eg：5)，但不能为0；也可以为百分数(eg：10%)。** + +如果未指定这两个可选参数，则k8s会使用默认配置： +``` bash +root@kube-aio:~# kubectl get deploy busy -o yaml +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + deployment.kubernetes.io/revision: "3" + creationTimestamp: 2018-08-19T02:42:56Z + generation: 3 + labels: + run: busy + name: busy + namespace: default + resourceVersion: "199461" + selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/busy + uid: 93fde307-a359-11e8-a93b-525400c61543 +spec: + progressDeadlineSeconds: 600 + replicas: 10 + revisionHistoryLimit: 10 + selector: + matchLabels: + run: busy + strategy: + rollingUpdate: + maxSurge: 1 # 滚动更新中最多超过预期值的 pod数 + maxUnavailable: 1 # 滚动更新中最多不可用的 pod数 + type: RollingUpdate +... +``` +### 5.1. 浅析部署概况 +``` bash +# 初始状态 +root@kube-aio:~# kubectl get deploy busy +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +busy 10 10 10 10 1h + +# 再做一遍回退 +root@kube-aio:~# kubectl rollout undo deploy busy +deployment.apps "busy" + +# 更新过程1 +root@kube-aio:~# kubectl get deploy busy +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +busy 10 11 2 9 1h + +# 更新过程2 +root@kube-aio:~# kubectl get deploy busy +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +busy 10 11 4 9 1h + +# 更新过程3 +root@kube-aio:~# kubectl get deploy busy +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +busy 10 11 6 9 1h + +# 更新结束 +root@kube-aio:~# kubectl get deploy busy +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +busy 10 10 10 10 1h +``` +>* `DESIRED`   最终期望处于READY状态的副本数   +>* `CURRENT` 当前的副本总数 +>* `UP-TO-DATE` 当前完成更新的副本数 +>* `AVAILABLE` 当前可用的副本数 + +当前的副本总数：10(DESIRED) + 1(maxSurge) = 11，所以CURRENT为11。 +当前可用的副本数：10(DESIRED) - 1(maxUnavailable) = 9，所以AVAILABLE为9。 + +### 5.2. 浅析部署详情 + +``` bash +root@kube-aio:~# kubectl describe deploy busy +Name: busy +Namespace: default +CreationTimestamp: Sun, 19 Aug 2018 12:27:19 +0800 +Labels: run=busy +Annotations: deployment.kubernetes.io/revision=2 +Selector: run=busy +Replicas: 10 desired | 10 updated | 10 total | 10 available | 0 unavailable +StrategyType: RollingUpdate +MinReadySeconds: 0 +RollingUpdateStrategy: 1 max unavailable, 1 max surge +Pod Template: + Labels: run=busy + Containers: + busy: + Image: busybox:1.29 + Port: + Host Port: + Args: + sleep + 3600000 + Environment: + Mounts: + Volumes: +Conditions: + Type Status Reason + ---- ------ ------ + Available True MinimumReplicasAvailable + Progressing True NewReplicaSetAvailable +OldReplicaSets: +NewReplicaSet: busy-84cb46955d (10/10 replicas created) +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal ScalingReplicaSet 1m deployment-controller Scaled up replica set busy-9669c8599 to 10 + Normal ScalingReplicaSet 46s deployment-controller Scaled up replica set busy-84cb46955d to 1 + Normal ScalingReplicaSet 46s deployment-controller Scaled down replica set busy-9669c8599 to 9 + Normal ScalingReplicaSet 46s deployment-controller Scaled up replica set busy-84cb46955d to 2 + Normal ScalingReplicaSet 43s deployment-controller Scaled down replica set busy-9669c8599 to 8 + Normal ScalingReplicaSet 43s deployment-controller Scaled up replica set busy-84cb46955d to 3 + Normal ScalingReplicaSet 43s deployment-controller Scaled down replica set busy-9669c8599 to 7 + Normal ScalingReplicaSet 43s deployment-controller Scaled up replica set busy-84cb46955d to 4 + Normal ScalingReplicaSet 40s deployment-controller Scaled down replica set busy-9669c8599 to 6 + Normal ScalingReplicaSet 28s (x12 over 40s) deployment-controller (combined from similar events): Scaled down replica set busy-9669c8599 to 0 +``` +整个滚动过程是通过控制两个副本集来完成的，新的副本集：busy-84cb46955d；旧的副本集：busy-9669c8599 。 +理想状态下的滚动过程： +>1. 创建新副本集，并为其分配1个新版本的pod。 +>2. 通知旧副本集，销毁1个旧版本的pod。 +>3. 当旧副本销毁成功后，通知新副本集，再新增1个新版本的pod；当新副本创建成功后，通知旧副本再减少1个pod。 +>只要销毁成功，新副本集就会创造新的pod，一直循环，直到旧的副本集pod数量为0。 +### 5.4 总结 +**`无论理想还是不理想，k8s最终都会使应用程序全部更新到期望状态，都会始终保持最大的副本总数和可用副本总数的不变性！！！`** + +[阅读原文](http://www.cnblogs.com/justmine/p/8688828.html) + diff --git a/docs/mixes/01.fix_kubelet_annoymous_access.md b/docs/mixes/01.fix_kubelet_annoymous_access.md new file mode 100644 index 0000000..965dbd4 --- /dev/null +++ b/docs/mixes/01.fix_kubelet_annoymous_access.md @@ -0,0 +1,24 @@ +# 修复kubelet默认允许匿名访问 + +kubelet默认启动参数`--anonymous-auth=true`风险非常大，黑客可以在集群中植入挖坑程序，甚至通过这个漏洞获取宿主系统root权限。感谢 `cqspirit` [PR #192](https://github.com/gjmzj/kubeasz/pull/192) 提醒 + +## 关于漏洞的危害 + +据我所知k8s v1.5+ 所有版本的kubelet组件的默认启动参数是允许匿名访问kubelet的（默认的大坑），你可以使用如下命令检查你的集群： +`curl -sk https://$NodeIP:10250/runningpods/` +- 如果返回了运行的pod信息，说明是允许匿名访问的， +- 如果返回`Unauthorized`，说明是安全的 + +部分关于该漏洞的讨论参考如下： +- [kubelet-exploit](https://github.com/kayrus/kubelet-exploit) +- [Kubernetes-From-Container-To-Cluster](https://raesene.github.io/blog/2016/10/08/Kubernetes-From-Container-To-Cluster/) +- [Analysis of a Kubernetes hack -- Backdooring through kubelet](https://www.reddit.com/r/netsec/comments/847994/analysis_of_a_kubernetes_hack_backdooring_through/) + +## 漏洞的修复 + +最新代码已经修复，参考[官方文档说明](https://kubernetes.io/docs/admin/kubelet-authentication-authorization/)，已有集群可以登陆`deploy`节点操作如下： +``` bash +$ cd /etc/ansible +$ git pull origin master +$ ansible-playbook 22.upgrade.yml -t restart_master,restart_node +``` diff --git a/docs/mixes/DoneList.md b/docs/mixes/DoneList.md new file mode 100644 index 0000000..0aefad6 --- /dev/null +++ b/docs/mixes/DoneList.md @@ -0,0 +1,55 @@ +## 前言 + +`kubeasz`项目开始于`2017.11`，半年多时间以来，从最开始单一的ansible部署脚本朝着提供部署高可用 K8S集群的完整解决方案的目标不断前进，接下去项目的发展需要各位的共同参与和贡献，希望越做越好，为国内k8s学习、实践者提供更多帮助。 + +### 项目已完成部分 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类型描述备注
集群部署服务器基础安全加固与参数优化已完成
基础服务集群监控告警-prometheus已完成基础，待优化
应用服务jenkins集成已完成
集群部署kube-router网络插件已完成
基础服务metrics server已完成
集群部署ipvs代理模式跟进已完成
集群部署cilium网络插件已完成
集群部署集群内时间同步-Chrony已完成
+ + diff --git a/docs/mixes/HowToContribute.md b/docs/mixes/HowToContribute.md new file mode 100644 index 0000000..2c7ffc7 --- /dev/null +++ b/docs/mixes/HowToContribute.md @@ -0,0 +1,19 @@ +# 为项目`kubeasz`提交`pull request` + +首先请核对下本地git config配置的用户名和邮箱与你github上的注册用户和邮箱一致，否则即使`pull request`被接受，贡献者列表中也看不到自己的名字，设置命令： + +``` bash +$ git config --global user.email "you@example.com" +$ git config --global user.name "Your Name" +``` + +- 1.登陆github，在本项目页面点击`fork`到自己仓库 +- 2.clone 自己的仓库到本地：`git clone https://github.com/xxx/kubeasz.git` +- 3.在 master 分支添加原始仓库为上游分支：`git remote add upstream https://github.com/gjmzj/kubeasz.git` +- 4.在本地新建开发分支：`git checkout -b dev` +- 5.在开发分支修改代码并提交：`git add .`, `git commit -am 'xx变更说明'` +- 6.切换至 master 分支，同步原始仓库：`git checkout master`， `git pull upstream master` +- 7.切换至 dev 分支，合并本地 master 分支（已经和原始仓库同步），可能需要解冲突：`git checkout dev`, `git merge master` +- 8.提交本地 dev 分支到自己的远程 dev 仓库：`git push origin dev` +- 9.在github自己仓库页面，点击`Compare & pull request`给原始仓库发 pull request 请求 +- a.等待原作者回复（接受/拒绝） diff --git a/docs/mixes/LICENSE b/docs/mixes/LICENSE new file mode 100644 index 0000000..48a6741 --- /dev/null +++ b/docs/mixes/LICENSE @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "{}" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright 2017 jmgao + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/docs/mixes/TodoList.md b/docs/mixes/TodoList.md new file mode 100644 index 0000000..80729af --- /dev/null +++ b/docs/mixes/TodoList.md @@ -0,0 +1,31 @@ +## 前言 + +`kubeasz`项目开始于`2017.11`，从最开始单一的ansible部署脚本朝着提供部署高可用 K8S集群的完整解决方案的目标不断前进；项目的发展需要各位的共同参与和贡献，希望越做越好，为国内k8s学习、实践者提供更多帮助。 +现在是很好的时间节点，梳理项目的`TodoList`/`Milestones`，以下是现有的一些想法，欢迎大家在[这里](https://github.com/gjmzj/kubeasz/issues/188)参与讨论。 + +### 项目 TodoList + + + + + + + + + + + + + + + + + + + + + + +
类型描述备注
基础服务集群监控告警-prometheus已完成基础，待优化
应用服务服务治理-istio跟进
基础服务ingress-nginx待集成manifests及使用文档
+ + diff --git a/docs/mixes/branch.md b/docs/mixes/branch.md new file mode 100644 index 0000000..ec6aee7 --- /dev/null +++ b/docs/mixes/branch.md @@ -0,0 +1,18 @@ +# 项目分支说明 + +- 为减小维护压力，避免产生混淆，精简项目分支为：`master`, `dev`, `closed`, `release-0`; 原分支`v1.9`取消（可以直接更新至master分支），原分支`v1.8`更名为 `closed`并停止更新。 +- 更新： + - kubeasz 0.6 及之前的版本已合并至 release-0 分支，后续停止更新，只做 bug 修复 + - kubeasz 1.x 版本将在 master 分支更新发布 + +## 项目分支与百度网盘离线包关系 + +- `master` 分支目前支持k8s v1.8/v1.9/v1.10/v1.11/v1.12/v1.13 的集群安装，请使用 `k8s.1-x-x.tar.gz` 的安装包 +- `dev` 分支同 `master`的离线安装包 +- `closed` 分支请使用分享链接目录`old/`下的 `k8s.18x.tar.gz` 的安装包（已停止更新） + +## 更新频率和内容 + +- `master` 为最新稳定分支，更新相对频繁：**相关文档**，**功能特性**，BUG修复，组件更新 +- `dev` 为开发测试分支，不一定稳定，更新频繁：**相关文档**，**功能特性**，BUG修复，组件更新 +- `closed` 分支已停止更新：仅重要BUG修复，留档保存使用 diff --git a/docs/mixes/donate.md b/docs/mixes/donate.md new file mode 100644 index 0000000..d705986 --- /dev/null +++ b/docs/mixes/donate.md @@ -0,0 +1,8 @@ +# 捐赠 + +如果觉得本项目对您有帮助，请小小鼓励下项目作者，谢谢！ + +支付宝码(左)和微信钱包码(右) + +![支付宝](../../pics/alipay.gif) ![微信钱包](../../pics/wxpay.gif) + diff --git a/docs/op/AddMaster.md b/docs/op/AddMaster.md new file mode 100644 index 0000000..5905456 --- /dev/null +++ b/docs/op/AddMaster.md @@ -0,0 +1,43 @@ +## 增加 kube-master 节点 + +注意：目前仅支持按照本项目`多主模式`(hosts.m-masters.example/hosts.cloud.example)部署的`k8s`集群增加`master`节点 + +新增`kube-master`节点大致流程为： +- [可选]新节点安装 chrony 时间同步 +- 新节点预处理 prepare +- 新节点安装 docker 服务 +- 新节点安装 kube-master 服务 +- 新节点安装 kube-node 服务 +- 新节点安装网络插件相关 +- 禁止业务 pod调度到新master节点 +- 更新配置 haproxy 负载均衡并重启 + +### 操作步骤 + +首先配置 ssh 免密码登陆新增节点，然后执行 (假设待增加节点为 192.168.1.11)： + +``` bash +$ easzctl add-master 192.168.1.11 +``` + +### 验证 + +``` bash +# 在新节点master 服务状态 +$ systemctl status kube-apiserver +$ systemctl status kube-controller-manager +$ systemctl status kube-scheduler + +# 查看新master的服务日志 +$ journalctl -u kube-apiserver -f + +# 查看集群节点，可以看到新 master节点 Ready, 并且禁止了POD 调度功能 +$ kubectl get node +NAME STATUS ROLES AGE VERSION +192.168.1.1 Ready,SchedulingDisabled 3h v1.9.3 +192.168.1.2 Ready,SchedulingDisabled 3h v1.9.3 +192.168.1.3 Ready 3h v1.9.3 +192.168.1.4 Ready 3h v1.9.3 +192.168.1.11 Ready,SchedulingDisabled 2h v1.9.3 # 新增 master节点 +``` + diff --git a/docs/op/AddNode.md b/docs/op/AddNode.md new file mode 100644 index 0000000..4ca27e3 --- /dev/null +++ b/docs/op/AddNode.md @@ -0,0 +1,29 @@ +## 增加 kube-node 节点 + +新增`kube-node`节点大致流程为： +- [可选]新节点安装 chrony 时间同步 +- 新节点预处理 prepare +- 新节点安装 docker 服务 +- 新节点安装 kube-node 服务 +- 新节点安装网络插件相关 + +### 操作步骤 + +首先配置 ssh 免密码登陆新增节点，然后执行 (假设待增加节点为 192.168.1.11)： + +``` bash +$ easzctl add-node 192.168.1.11 +``` + +### 验证 + +``` bash +# 验证新节点状态 +$ kubectl get node + +# 验证新节点的网络插件calico 或flannel 的Pod 状态 +$ kubectl get pod -n kube-system + +# 验证新建负载能否调度到新节点，略 +``` + diff --git a/docs/op/ChangeVIP.md b/docs/op/ChangeVIP.md new file mode 100644 index 0000000..31175b6 --- /dev/null +++ b/docs/op/ChangeVIP.md @@ -0,0 +1,107 @@ +# 更改高可用 `Master IP` + +**WARNING:** 更改集群的 `Master VIP`操作有风险，不建议在生产环境直接操作，此文档实践一个修改的操作流程，帮助理解整个集群运行架构和 `kubeasz`的部署逻辑，请在测试环境操作练手。 +**BUG:** 目前该操作只适用于集群网络选用`calico`，如果使用`flannel`操作变更后会出现POD地址分配错误的BUG。 + +首先分析大概操作思路： + +- 修改`/etc/ansible/hosts`里面的配置项`MASTER_IP` `KUBE_APISERVER` +- 修改LB节点的keepalive的配置，重启keepalived服务 +- 修改kubectl/kube-proxy的配置文件，使用新VIP地址更新api-server地址 +- 重新生成master证书，hosts字段包含新VIP地址 +- 修改kubelet的配置文件（kubelet的配置文件和证书是由bootstrap机制自动生成的） + - 删除kubelet.kubeconfig + - 删除集群所有node 节点 + - 所有节点重新bootstrap + +## 变更前状态验证 + +``` bash +$ kubectl get cs,node,pod -o wide +NAME STATUS MESSAGE ERROR +controller-manager Healthy ok +scheduler Healthy ok +etcd-2 Healthy {"health":"true"} +etcd-0 Healthy {"health":"true"} +etcd-1 Healthy {"health":"true"} + +NAME STATUS ROLES AGE VERSION EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +192.168.1.41 Ready,SchedulingDisabled 2h v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-97-generic docker://18.3.0 +192.168.1.42 Ready,SchedulingDisabled 2h v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-97-generic docker://18.3.0 +192.168.1.43 Ready 2h v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-97-generic docker://18.3.0 +192.168.1.44 Ready 2h v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-98-generic docker://18.3.0 +192.168.1.45 Ready 2h v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-98-generic docker://18.3.0 + +NAME READY STATUS RESTARTS AGE IP NODE +busy-5d6b6b5d4b-8wxkp 1/1 Running 0 17h 172.20.135.133 192.168.1.41 +busy-5d6b6b5d4b-fcmkp 1/1 Running 0 17h 172.20.135.128 192.168.1.41 +busy-5d6b6b5d4b-ptvd7 1/1 Running 0 17h 172.20.135.136 192.168.1.41 +nginx-768979984b-ncqbp 1/1 Running 0 17h 172.20.135.137 192.168.1.41 + +# 查看待变更集群 Master VIP +$ kubectl cluster-info +Kubernetes master is running at https://192.168.1.39:8443 +``` + +## 变更操作 + +- `ansible playbook`可以使用tags来控制只允许部分任务执行，这里为简化操作没有细化，在deploy节点具体操作如下： + +``` bash +# 1.修改/etc/ansible/hosts 配置项MASTER_IP，KUBE_APISERVER + +# 2.删除集群所有node节点，等待重新bootstrap +$ kubectl get node |grep Ready|awk '{print $1}' |xargs kubectl delete node + +# 3.重置keepalived 和修改kubectl/kube-proxy/bootstrap配置 +$ ansible-playbook 01.prepare.yml + +# 4.删除旧master证书 +$ ansible kube-master -m file -a 'path=/etc/kubernetes/ssl/kubernetes.pem state=absent' + +# 5.删除旧kubelet配置文件 +$ ansible all -m file -a 'path=/etc/kubernetes/kubelet.kubeconfig state=absent' + +# 6.重新配置启动master节点 +$ ansible-playbook 04.kube-master.yml + +# 7.重新配置启动node节点 +$ ansible-playbook 05.kube-node.yml +``` + +## 变更后验证 + +``` bash +$ kubectl get cs,node,pod -o wide +NAME STATUS MESSAGE ERROR +scheduler Healthy ok +controller-manager Healthy ok +etcd-2 Healthy {"health":"true"} +etcd-1 Healthy {"health":"true"} +etcd-0 Healthy {"health":"true"} + +NAME STATUS ROLES AGE VERSION EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +192.168.1.41 Ready,SchedulingDisabled 4m v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-97-generic docker://18.3.0 +192.168.1.42 Ready,SchedulingDisabled 4m v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-97-generic docker://18.3.0 +192.168.1.43 Ready 3m v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-97-generic docker://18.3.0 +192.168.1.44 Ready 3m v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-98-generic docker://18.3.0 +192.168.1.45 Ready 3m v1.10.0 Ubuntu 16.04.3 LTS 4.4.0-98-generic docker://18.3.0 + +NAME READY STATUS RESTARTS AGE IP NODE +busy-5d6b6b5d4b-25hfr 1/1 Running 0 5m 172.20.237.64 192.168.1.43 +busy-5d6b6b5d4b-cdzb5 1/1 Running 0 5m 172.20.145.192 192.168.1.44 +busy-5d6b6b5d4b-m2rf7 1/1 Running 0 5m 172.20.26.131 192.168.1.45 +nginx-768979984b-2ngww 1/1 Running 0 5m 172.20.145.193 192.168.1.44 + +# 查看集群master VIP已经变更 +$ kubectl cluster-info +Kubernetes master is running at https://192.168.1.40:8443 +``` + +## 小结 + +本示例操作演示了多主多节点k8s集群变更`Master VIP`的操作，有助于理解整个集群组件架构和`kubeasz`的安装逻辑，小结如下： + +- 变更操作不影响集群已运行的业务POD，但是操作过程中业务会中断 +- 已运行POD会重新调度到各node节点，如果业务POD量很大，短时间内会对集群造成压力 +- 不建议在生成环境直接操作，本示例演示说明为主 diff --git a/docs/op/ch_apiserver_cert.md b/docs/op/ch_apiserver_cert.md new file mode 100644 index 0000000..626d554 --- /dev/null +++ b/docs/op/ch_apiserver_cert.md @@ -0,0 +1,41 @@ +# 修改 APISERVER（MASTER）证书 + +`kubeasz` 创建集群后，APISERVER（MASTER）证书默认 CN 包含如下`域名`和`IP`：参见`roles/kube-master/templates/kubernetes-csr.json.j2` + +``` + "hosts": [ + "127.0.0.1", + "{{ MASTER_IP }}", + "{{ inventory_hostname }}", + "{{ CLUSTER_KUBERNETES_SVC_IP }}", +{% for HOST in MASTER_CERT_HOSTS %} + "{{ HOST }}", +{% endfor %} + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster", + "kubernetes.default.svc.cluster.local" + ], +``` + +有的时候（比如apiserver地址通过边界防火墙的NAT转换成公网IP访问，或者需要添加公网域名访问）我们需要在 APISERVER（MASTER）证书中添加一些`域名`或者`IP`，可以方便操作如下： + +## 1.修改配置文件`roles/kube-master/defaults/main.yml` + +``` bash +# k8s 集群 master 节点证书配置，可以添加多个ip和域名（比如增加公网ip和域名） +MASTER_CERT_HOSTS: + - "10.1.1.1" + - "k8s.test.io" + #- "61.182.11.41" + #- "www.test.com" +``` + +## 2.执行新证书生成即可 + +``` bash +$ ansible-playbook 04.kube-master.yml -t change_cert +``` + +- 注：新证书生效无需重启`kube-apiserver.service`服务 diff --git a/docs/op/change_ip_allinone.md b/docs/op/change_ip_allinone.md new file mode 100644 index 0000000..3c0c5d8 --- /dev/null +++ b/docs/op/change_ip_allinone.md @@ -0,0 +1,17 @@ +# 修改AIO 部署的IP +前两天在项目[ISSUES #201](https://github.com/gjmzj/kubeasz/issues/201)看到有人提：`在虚拟机A装了allinone，并搭建一套开发环境，我想通过copy A出来一套B然后交给别人测试`，觉得这个场景蛮有用，就写了这个文档和对应的脚本，希望对各位有帮助，也可以熟悉kubeasz的安装逻辑。 + +首先，因为kubeasz创建的集群都是TLS双向认证的，所以修改host ip地址比想象中要复杂很多。具体步骤可以参考[脚本](../../tools/change_ip_aio.yml)中的注释内容。 + +- 本操作指南仅适用于测试交流 + +## 操作步骤 +前提 ：一个运行正常的allinone部署在虚机，关机后复制给别人使用，新虚机开机后如果需要修改IP，请执行如下步骤： + +- 0.拉取最新项目代码：`git pull origin master` +- 1.修改ansible hosts文件：`sed -i 's/$OLD_IP/$NEW_IP/g' /etc/ansible/hosts` +- 2.配置ssh免密码登陆：`ssh-copy-id $NEW_IP` 按提示完成 +- 3.检查下修改是否成功，并且能够成功执行 `ansible all -m ping` +- 4.以上步骤完成后，执行 `ansible-playbook /etc/ansible/tools/change_ip_aio.yml` + +执行成功即可，请自己验证原先集群中各应用是否正常。 diff --git a/docs/op/change_k8s_network.md b/docs/op/change_k8s_network.md new file mode 100644 index 0000000..0d402fc --- /dev/null +++ b/docs/op/change_k8s_network.md @@ -0,0 +1,31 @@ +# 替换k8s集群的网络插件 + +有时候我们在测试环境的k8s集群中希望试用多种网络插件（calico/flannel/kube-router），又不希望每测试一次就全部清除集群然后重建，那么可能这个文档适合你。 +- WARNNING：重新安装k8s网络插件会短暂中断已有运行在k8s上的服务 + - 请在熟悉kubeasz的安装流程和k8s网络插件安装流程的基础上谨慎操作 + - 如果k8s集群已经运行庞大业务pod，重装网络插件时会引起所有pod的删除、重建，短时间内将给apiserver带来压力，可能引起master节点夯住 + - 确保没有裸pod 运行(因为最后需要删除所有pod 重建，裸pod 不会重建)，即所有pod 都是由 deploy/daemonset/statefulset 等创建； + +## 替换流程 + +kubeasz使用标准cni方式安装k8s集群的网络插件；cni负载创建容器网卡和IP分配（IPAM），不同的网络插件（calico,flannel等）创建容器网卡和IP分配方式不一样，所以在替换网络插件时候需要现有pod全部删除，然后自动按照新网络插件的方式重建pod网络；请参考[k8s网络插件章节](../setup/06-install_network_plugin.md)。 + +### 替换操作 + +替换网络插件操作很简单，只要两步： +- 1.修改ansible hosts文件指定新网络插件 +- 2.执行替换脚本 `ansible-playbook /etc/ansible/tools/change_k8s_network.yml` + +对照脚本`change_k8s_network.yml` 讲解下大致流程为： +a.根据实际运行情况，删除现有网络组件的daemonset pod +b.如果现有组件是kube-router 需要进行一些额外清理 +c.暂停node相关服务，后面才可以进一步清理iptables等 +d.执行旧网络插件相关清理 +e.重新开启node相关服务 +f.安装新网络插件 +g.删除所有运行pod，然后等待自动重建 + +## 验证新网络插件 + +参照[calico](../setup/network-plugin/calico.md) [cilium](../setup/network-plugin/cilium.md) [flannel](../setup/network-plugin/flannel.md) [kube-router](../setup/network-plugin/kube-router.md) + diff --git a/docs/op/clean_one_node.md b/docs/op/clean_one_node.md new file mode 100644 index 0000000..e532898 --- /dev/null +++ b/docs/op/clean_one_node.md @@ -0,0 +1,42 @@ +# 删除节点 + +本文档所指删除的节点是指使用kubeasz项目安装的节点角色（可能是kube-master, kube-node, etcd, lb节点） + +- 警告：此操作将清理单个node节点，包含k8s集群可能使用的数据，特别的：如果有pod使用了本地存储类型，请自行判断重要性 + +## 删除流程解释 + +- 0.判断待删除节点不是 etcd/master 组的唯一节点，否则不允许删除 +- 1.待删除节点可能是kube-node节点，因此先执行`kubectl drain`，如果不是忽略执行报错 +- 2.参照`99.clean.yml`脚本方式删除节点可能的服务和配置，忽略执行报错 +- 3.待删除节点可能是kube-node节点，执行`kubectl delete node`, 如果不是忽略执行报错 +- 4.修改ansible hosts，移除删除节点 + +## 删除操作 + +可以使用以下三种方式删除节点（i.e. 192.168.1.1） + +``` bash +# 1.推荐使用 easzctl 工具 +$ easzctl clean-node 192.168.1.1 + +# 2.ansible-playbook 带参数执行如下 +$ ansible-playbook /etc/ansible/tools/clean_one_node.yml -e NODE_TO_DEL=192.168.1.1 + +# 3.ansible-playbook 不带参数执行，然后根据提示输入/确认 +$ ansible-playbook /etc/ansible/tools/clean_one_node.yml +``` + +## 验证 + +- 验证删除节点上是否相关服务均已停止 +- 验证 ansible hosts 文件中已删除节点 + +## Debug + +如果出现清理失败，类似报错：`... Device or resource busy: '/var/run/docker/netns/xxxxxxxxxx'`，需要手动umount该目录后重新清理 + +``` bash +$ umount /var/run/docker/netns/xxxxxxxxxx +$ ansible-playbook /etc/ansible/tools/clean_one_node.yml +``` diff --git a/docs/op/cluster_restore.md b/docs/op/cluster_restore.md new file mode 100644 index 0000000..967e580 --- /dev/null +++ b/docs/op/cluster_restore.md @@ -0,0 +1,129 @@ +# K8S 集群备份与恢复 + +虽然 K8S 集群可以配置成多主多节点的高可用的部署，还是有必要了解下集群的备份和容灾恢复能力；在高可用k8s集群中 etcd集群保存了整个集群的状态，因此这里的备份与恢复重点就是： + +- 从运行的etcd集群备份数据到磁盘文件 +- 从etcd备份文件恢复数据到运行的etcd集群，然后据此重建整个集群 + +## 前提 + +k8s 集群可能因为软硬件故障或者误操作出现了不可自愈的问题，这个时候需要考虑集群从备份中恢复重建；使用kubeasz项目创建的集群如需恢复前提如下： + +- 集群正常状态下的etcd 备份文件（etcd V3数据） +- 创建集群时使用的 CA证书相关文件 +- 创建集群时使用的 ansible hosts文件 + +## 备份与恢复手动操作说明 + +首先用kubeasz 搭建一个测试集群，部署几个测试deployment，验证集群各项正常后，进行一次备份： + +- 1.在一个etcd节点上执行数据备份，把产生的备份文件`snapshot.db`复制到所有etcd集群节点 + +``` bash +$ mkdir -p /backup/k8s/ && cd /backup/k8s +$ ETCDCTL_API=3 etcdctl snapshot save snapshot.db +``` + +- 2.在deploy节点把 CA证书相关备份出来 + +``` bash +$ mkdir -p /backup/k8s/ && cp /etc/kubernetes/ssl/ca* /backup/k8s/ +``` + +- 3.在deploy节点清理集群，模拟集群完全崩溃 + +``` bash +$ ansible-playbook /etc/ansible/99.clean.yml +``` + +- 4.在deploy节点开始一步步重建集群 + +``` bash +# 恢复原集群的CA 证书相关 +$ mkdir -p /etc/kubernetes/ssl/ && cp /backup/k8s/* /etc/kubernetes/ssl/ + +# 然后执行集群恢复步骤，安装至 kube-node完成阶段 +$ cd /etc/ansible +$ ansible-playbook 01.prepare.yml +$ ansible-playbook 02.etcd.yml +$ ansible-playbook 03.docker.yml +$ ansible-playbook 04.kube-master.yml +$ ansible-playbook 05.kube-node.yml + +# 以上步骤验证正常后，停止etcd集群服务，并清空新etcd集群数据目录 +$ ansible etcd -m service -a 'name=etcd state=stopped' +$ asnible etcd -m file -a 'name=/var/lib/etcd/member/ state=absent' +``` + +- 5.手动分别登陆每个etcd节点进行数据备份恢复，每个etcd都要如下操作 + +``` bash +# 参照本etcd节点/etc/systemd/system/etcd.service的服务文件，替换如下{{}}中变量后执行 +$ cd /backup/k8s/ +$ ETCDCTL_API=3 etcdctl snapshot restore snapshot.db \ + --name {{ NODE_NAME }} \ + --initial-cluster {{ ETCD_NODES }} \ + --initial-cluster-token etcd-cluster-0 \ + --initial-advertise-peer-urls https://{{ inventory_hostname }}:2380 + +# 以上执行完后，会生成{{ NODE_NAME }}.etcd的文件夹，将它里面的member 拷贝到etcd数据目录中 +$ cp -r {{ NODE_NAME }}.etcd/member /var/lib/etcd/ + +$ systemctl restart etcd +``` + +- 6.在deploy节点执行网络重建 + +``` bash +$ ansible-playbook /etc/ansible/tools/change_k8s_network.yml +``` + +执行完之后，可以验证整个集群是否恢复正常，之前的测试应用部署是否全部恢复。 + +- 参考：https://github.com/coreos/etcd/blob/master/Documentation/op-guide/recovery.md + +## 备份恢复自动脚本操作指南 + +- 一.集群备份 + +``` bash +$ ansible-playbook /etc/ansible/23.backup.yml +``` + +执行完毕可以在目录 `/etc/ansible/roles/cluster-backup/files`下检查备份情况，示例如下： + +``` bash +roles/cluster-backup/files/ +├── ca # 集群CA 相关备份 +│   ├── ca-config.json +│   ├── ca.csr +│   ├── ca-csr.json +│   ├── ca-key.pem +│   └── ca.pem +├── hosts # ansible hosts备份 +│   ├── hosts # 最近的备份 +│   └── hosts-201807231642 +├── readme.md +└── snapshot # etcd 数据备份 + ├── snapshot-201807231642.db + └── snapshot.db # 最近的备份 +``` + +- 二.模拟集群故障 + +``` bash +$ ansible-playbook /etc/ansible/99.clean.yml +``` + +**注意** 为了模拟集群彻底崩溃，这里清理整个集群；实际操作中，在有备份前提下，也建议彻底清理集群后再尝试去恢复 + +- 三.集群恢复 + +可以在 `roles/cluster-restore/defaults/main.yml` 文件中配置需要恢复的 etcd备份版本，默认使用最近一次备份 + +``` bash +$ ansible-playbook /etc/ansible/24.restore.yml +$ ansible-playbook /etc/ansible/tools/change_k8s_network.yml +``` + +执行完成可以验证整个集群是否恢复如初！ diff --git a/docs/op/loadballance_ingress_nodeport.md b/docs/op/loadballance_ingress_nodeport.md new file mode 100644 index 0000000..67d80fd --- /dev/null +++ b/docs/op/loadballance_ingress_nodeport.md @@ -0,0 +1,54 @@ +# 配置负载转发 ingress nodeport + +向集群外暴露 ingress-controller 本身的服务端口（80/443/8080）一般有以下三种方法： + +- 1.部署ingress-controller时使用`hostNetwork: true`，这样就可以直接使用上述端口，可能与host已listen端口冲突 +- 2.部署ingress-controller时使用`LoadBalancer`类型服务，需要集群支持`LoadBalancer` +- 3.部署ingress-controller时使用`nodePort`类型服务，然后在集群外使用 haproxy/f5 等配置 virtual server 集群 + +本文档讲解使用 haproxy 配置 ingress的 VS 集群，前提是`多主多节点集群`并且配置了自建`lb`节点 + +## 1.配置 lb 参数开启转发 ingress nodeport + +``` bash +# 编辑 roles/lb/defaults/main.yml，配置如下变量 +INGRESS_NODEPORT_LB: "yes" +INGRESS_TLS_NODEPORT_LB: "yes" +``` + +## 2.重新配置启动LB节点服务 + +``` bash +$ ansible-playbook /etc/ansible/roles/lb/lb.yml +``` + +## 3.验证 lb 节点的 haproxy 服务配置 `/etc/haproxy/haproxy.cfg` 包含如下配置 + +``` bash +... 前文省略 +listen kube-master + bind 0.0.0.0:8443 + mode tcp + option tcplog + balance roundrobin + server 192.168.1.1 192.168.1.1:6443 check inter 2000 fall 2 rise 2 weight 1 + server 192.168.1.2 192.168.1.2:6443 check inter 2000 fall 2 rise 2 weight 1 + +listen ingress-node + bind 0.0.0.0:80 + mode tcp + option tcplog + balance roundrobin + server 192.168.1.3 192.168.1.3:23456 check inter 2000 fall 2 rise 2 weight 1 + server 192.168.1.4 192.168.1.4:23456 check inter 2000 fall 2 rise 2 weight 1 + +listen ingress-node-tls + bind 0.0.0.0:443 + mode tcp + option tcplog + balance roundrobin + server 192.168.1.3 192.168.1.3:23457 check inter 2000 fall 2 rise 2 weight 1 + server 192.168.1.4 192.168.1.4:23457 check inter 2000 fall 2 rise 2 weight 1 +``` + +验证成功后，我们可以方便的去做[配置ingress](../guide/ingress.md)和[配置https ingress](../guide/ingress-tls.md)实验了。 diff --git a/docs/op/op-etcd.md b/docs/op/op-etcd.md new file mode 100644 index 0000000..f843582 --- /dev/null +++ b/docs/op/op-etcd.md @@ -0,0 +1,53 @@ +# 管理 etcd 集群 + +Etcd 集群支持在线改变集群成员节点，可以增加、修改、删除成员节点；不过改变成员数量仍旧需要满足集群成员多数同意原则（quorum），另外请记住集群成员数量变化的影响： + +- 增加 etcd 集群节点, 提高集群稳定性 +- 增加 etcd 集群节点, 提高集群读性能（所有节点数据一致，客户端可以从任意节点读取数据） +- 增加 etcd 集群节点, 降低集群写性能（所有节点数据一致，每一次写入会需要所有节点数据同步） + +## 备份 etcd 数据 + +可以根据需要进行定期备份（使用 crontab），或者手动在任意正常 etcd 节点上执行备份： + +``` bash +# snapshot备份 +$ ETCDCTL_API=3 etcdctl snapshot save backup.db +# 查看备份 +$ ETCDCTL_API=3 etcdctl --write-out=table snapshot status backup.db +``` + +## etcd 集群节点操作 + +首先确认配置 ssh 免密码登陆，然后执行 (假设待操作节点为 192.168.1.11)： + +- 增加 etcd 节点：`$ easzctl add-etcd 192.168.1.11` (注意：增加 etcd 还需要根据提示输入集群内唯一的 NODE_NAME) +- 删除 etcd 节点：`$ easzctl del-etcd 192.168.1.11` + +### 验证 etcd 集群 + +``` bash +# 登陆任意etcd节点验证etcd集群状态 +$ export ETCDCTL_API=3 +$ etcdctl member list + +# 验证所有etcd节点服务状态和日志 +$ systemctl status etcd +$ journalctl -u etcd -f +``` + +### 重置 k8s 连接 etcd 参数 + +上述步骤验证成功，确认新etcd集群工作正常后，可以重新配置运行apiserver，以让 k8s 集群能够识别新的etcd集群： + +``` bash +# 重启 master 节点服务 +$ ansible-playbook /etc/ansible/04.kube-master.yml -t restart_master + +# 验证 k8s 能够识别新 etcd 集群 +$ kubectl get cs +``` + +## 参考 + +- 官方文档 https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/runtime-configuration.md diff --git a/docs/op/op-index.md b/docs/op/op-index.md new file mode 100644 index 0000000..dce778e --- /dev/null +++ b/docs/op/op-index.md @@ -0,0 +1,14 @@ +# 集群运维管理指南 operation guide + +- [集群添加 NODE 节点](AddNode.md) +- [集群添加 MASTER 节点](AddMaster.md) +- [集群添加 ETCD 节点](AddEtcd.md) +- [升级 K8S 版本](upgrade.md) +- [修改多主集群VIP地址](ChangeVIP.md) +- [修改AIO部署的系统IP](change_ip_allinone.md) +- [集群删除单个节点](clean_one_node.md) +- [替换集群使用的网络插件](change_k8s_network.md) +- [集群备份与恢复](cluster_restore.md) +- [设置只读权限 kubeconfig](readonly_kubectl.md) +- [修改 APISERVER 证书](ch_apiserver_cert.md) +- [配置负载转发 ingress nodeport](loadballance_ingress_nodeport.md) diff --git a/docs/op/readonly_kubectl.md b/docs/op/readonly_kubectl.md new file mode 100644 index 0000000..73d52ff --- /dev/null +++ b/docs/op/readonly_kubectl.md @@ -0,0 +1,70 @@ +# 配置 kubectl 只读访问权限 + +默认 k8s 集群安装后配置的 kubectl 客户端拥有所有的管理权限，而有时候我们需要把只读权限分发给普通开发人员，本文档将创建一个只读权限的kubectl 配置文档 kubeconfig。 + +## 创建 + +- 备份下原先 admin 权限的 kubeconfig 文件：`mv ~/.kube ~/.kubeadmin` +- 执行 `ansible-playbook /etc/ansible/roles/deploy/create-read-kubeconfig.yml`，成功后查看~/.kube/config 即为只读权限 + +## 讲解 + +对照文件`/etc/ansible/roles/deploy/create-read-kubeconfig.yml`，创建主要包括三个步骤： + +- 创建 group:read rbac 权限 +- 创建 read 用户证书和私钥 +- 创建 kubeconfig + +### read rbac 权限 + +所有权限控制魔法在`k8s`中由`rbac`实现，所谓`read`权限类似于集群自带的`clusterrole view`，具体查看： + +`kubectl get clusterrole view -o yaml` + +`read`权限配置`roles/deploy/files/read-group-rbac.yaml`是在`clusterrole view`基础上增加了若干读权限（Nodes/Persistent Volume Claims） + +### read 用户证书 + +准备 read 证书请求：`read-csr.json` + +``` bash +{ + "CN": "read", + "hosts": [], + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "CN", + "ST": "HangZhou", + "L": "XS", + "O": "group:read", + "OU": "System" + } + ] +} +``` +- 注意: O `group:read`，kube-apiserver 收到该证书后将请求的 Group 设置为`group:read`；之前步骤创建的 ClusterRoleBinding `read-clusterrole-binding`将 Group `group:read`与 ClusterRole `read-clusterrole`绑定，从而实现只读权限。 + +### read kubeconfig + +kubeconfig 为与apiserver交互使用的认证配置文件，如脚本步骤需要： + +- 设置集群参数，指定CA证书和apiserver地址 +- 设置客户端认证参数，指定使用read证书和私钥 +- 设置上下文参数，指定使用cluster集群和用户read +- 设置指定默认上下文 + +创建完成后生成默认配置文件为 `~/.kube/config` + +## 恢复 admin 权限 + +- 可以恢复之前备份的`~/.kubeadmin`文件：`mv ~/.kube ~/.kuberead && mv ~/.kubeadmin ~/.kube` +- 或者直接执行 `ansible-playbook /etc/ansible/roles/deploy/create-admin-kubeconfig.yml` + +## 参考 + +- [Using RBAC Authorization](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) +- [A Read Only Kubernetes Dashboard](https://blog.cowger.us/2018/07/03/a-read-only-kubernetes-dashboard.html) diff --git a/docs/op/upgrade.md b/docs/op/upgrade.md new file mode 100644 index 0000000..5657f30 --- /dev/null +++ b/docs/op/upgrade.md @@ -0,0 +1,47 @@ +## k8s 集群升级 + +集群升级存在一定风险，请谨慎操作。 + +- 项目分支`master`安装的集群可以在k8s 1.8/1.9/1.10/1.11/1.12/1.13 任意小版本、大版本间升级 +- 项目分支`closed`（已停止更新）安装的集群目前只能进行小版本1.8.x的升级 + +### 备份etcd数据 + +- 升级前对 etcd数据做备份，在任意 etcd节点上执行： + +``` bash +# snapshot备份 +$ ETCDCTL_API=3 etcdctl snapshot save backup.db +# 查看备份 +$ ETCDCTL_API=3 etcdctl --write-out=table snapshot status backup.db +``` +- `kubeasz`项目也可以方便执行 `ansible-playbook /etc/ansible/23.backup.yml`，详情阅读文档[备份恢复](cluster_restore.md) + +### 快速k8s版本升级 + +快速升级是指只升级`k8s`版本，比较常见如`Bug修复` `重要特性发布`时使用。 + +- 首先去官网release下载待升级的k8s版本，例如`https://dl.k8s.io/v1.11.5/kubernetes-server-linux-amd64.tar.gz` +- 解压下载的tar.gz文件，找到如下`kube*`开头的二进制，复制替换deploy节点目录`/etc/ansible/bin`对应文件 + - kube-apiserver + - kube-controller-manager + - kubectl + - kubelet + - kube-proxy + - kube-scheduler +- 在deploy节点执行`ansible-playbook -t upgrade_k8s 22.upgrade.yml`即可完成k8s 升级，不会中断业务应用 + +### 其他升级说明 + +其他升级是指升级k8s组件包括：`etcd版本` `docker版本`，一般不需要用到，以下仅作说明。 + +- 1.下载所有组件相关新的二进制解压并替换 `/etc/ansible/bin/` 目录下文件 + +- 2.升级 etcd: `ansible-playbook -t upgrade_etcd 02.etcd.yml`，**注意：etcd 版本只能升级不能降低！** + +- 3.升级 docker （建议使用k8s官方支持的docker稳定版本） + - 如果可以接受短暂业务中断，执行 `ansible-playbook -t upgrade_docker 03.docker.yml` + - 如果要求零中断升级，执行 `ansible-playbook -t download_docker 03.docker.yml`，然后手动执行如下 + - 待升级节点，先应用`kubectl cordon`和`kubectl drain`命令迁移业务pod + - 待升级节点执行 `systemctl restart docker` + - 恢复节点可调度 `kubectl uncordon` diff --git a/docs/practice/es_cluster.md b/docs/practice/es_cluster.md new file mode 100644 index 0000000..39b2ffd --- /dev/null +++ b/docs/practice/es_cluster.md @@ -0,0 +1,184 @@ +# Elasticsearch 部署实践 + +`Elasticsearch`是目前全文搜索引擎的首选，它可以快速地储存、搜索和分析海量数据；也可以看成是真正分布式的高效数据库集群；`Elastic`的底层是开源库`Lucene`；封装并提供了`REST API`的操作接口。 + +## 单节点 docker 测试安装 + +``` bash +cat > es-start.sh << EOF +#!/bin/bash + +sysctl -w vm.max_map_count=262144 + +docker run --detach \ + --name es01 \ + -p 9200:9200 -p 9300:9300 \ + -e "discovery.type=single-node" \ + -e "bootstrap.memory_lock=true" --ulimit memlock=-1:-1 \ + --ulimit nofile=65536:65536 \ + --volume /srv/elasticsearch/data:/usr/share/elasticsearch/data \ + --volume /srv/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml \ + jmgao1983/elasticsearch:6.4.0 +EOF +``` + +执行`sh es-start.sh`后，就在本地运行了。 + +- 验证 docker 镜像运行情况 + +``` bash +root@docker-ts:~# docker ps -a +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +171f3fecb596 jmgao1983/elasticsearch:6.4.0 "/usr/local/bin/do..." 2 hours ago Up 2 hours 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp es01 +``` + +- 验证 es 健康检查 + +``` bash +root@docker-ts:~# curl http://127.0.0.1:9200/_cat/health +epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent +1535523956 06:25:56 docker-es green 1 1 0 0 0 0 0 0 - 100.0% +``` + +## 在 k8s 上部署 Elasticsearch 集群 + +在生产环境下，Elasticsearch 集群由不同的角色节点组成： + +- master 节点：参与主节点选举，不存储数据；建议3个以上，维护整个集群的稳定可靠状态 +- data 节点：不参与选主，负责存储数据；主要消耗磁盘，内存 +- client 节点：不参与选主，不存储数据；负责处理用户请求，实现请求转发，负载均衡等功能 + +这里使用`helm chart`来部署 (https://github.com/helm/charts/tree/master/incubator/elasticsearch) + +- 1.安装 helm: 以本项目[安全安装helm](../guide/helm.md)为例 +- 2.准备 PV: 以本项目[K8S 集群存储](../setup/08-cluster-storage.md)创建`nfs`动态 PV 为例 + - 编辑配置文件：roles/cluster-storage/defaults/main.yml + +``` bash +storage: + nfs: + enabled: "yes" + server: "192.168.1.8" + server_path: "/share" + storage_class: "nfs-es" + provisioner_name: "nfs-provisioner-01" +``` + + - 创建 nfs provisioner + +``` bash +$ ansible-playbook /etc/ansible/roles/cluster-storage/cluster-storage.yml +# 执行成功后验证 +$ kubectl get pod --all-namespaces |grep nfs-prov +kube-system nfs-provisioner-01-6b7fbbf9d4-bh8lh 1/1 Running 0 1d +``` + +- 3.安装 elasticsearch chart + +``` bash +$ cd /etc/ansible/manifests/es-cluster +# 如果你的helm安装没有启用tls证书，请忽略以下--tls参数 +$ helm install --tls --name es-cluster --namespace elastic -f es-values.yaml elasticsearch +``` + +- 4.验证 es 集群 + +``` bash +# 验证k8s上 es集群状态 +$ kubectl get pod,svc -n elastic +NAME READY STATUS RESTARTS AGE +pod/es-cluster-elasticsearch-client-778df74c8f-7fj4k 1/1 Running 0 2m17s +pod/es-cluster-elasticsearch-client-778df74c8f-skh8l 1/1 Running 0 2m3s +pod/es-cluster-elasticsearch-data-0 1/1 Running 0 25m +pod/es-cluster-elasticsearch-data-1 1/1 Running 0 11m +pod/es-cluster-elasticsearch-master-0 1/1 Running 0 25m +pod/es-cluster-elasticsearch-master-1 1/1 Running 0 12m +pod/es-cluster-elasticsearch-master-2 1/1 Running 0 10m + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/es-cluster-elasticsearch-client NodePort 10.68.157.105 9200:29200/TCP,9300:29300/TCP 25m +service/es-cluster-elasticsearch-discovery ClusterIP None 9300/TCP 25m + +# 验证 es集群本身状态 +$ curl $NODE_IP:29200/_cat/health +1539335131 09:05:31 es-on-k8s green 7 2 0 0 0 0 0 0 - 100.0% + +$ curl $NODE_IP:29200/_cat/indices?v +health status index uuid pri rep docs.count docs.deleted store.size pri.store.size +root@k8s401:/etc/ansible# curl 10.100.97.41:29200/_cat/nodes? +172.31.2.4 27 80 5 0.09 0.11 0.21 mi - es-cluster-elasticsearch-master-0 +172.31.1.7 30 97 3 0.39 0.29 0.27 i - es-cluster-elasticsearch-client-778df74c8f-skh8l +172.31.3.7 20 97 3 0.11 0.17 0.18 i - es-cluster-elasticsearch-client-778df74c8f-7fj4k +172.31.1.5 8 97 5 0.39 0.29 0.27 di - es-cluster-elasticsearch-data-0 +172.31.2.5 8 80 3 0.09 0.11 0.21 di - es-cluster-elasticsearch-data-1 +172.31.1.6 18 97 4 0.39 0.29 0.27 mi - es-cluster-elasticsearch-master-2 +172.31.3.6 20 97 4 0.11 0.17 0.18 mi * es-cluster-elasticsearch-master-1 +``` + +### es 性能压测 + +如上已使用 chart 在 k8s上部署了 **7** 节点的 elasticsearch 集群；各位应该十分好奇性能怎么样；官方提供了压测工具[esrally](https://github.com/elastic/rally)可以方便的进行性能压测，这里省略安装和测试过程；压测机上执行： +`esrally --track=http_logs --target-hosts="$NODE_IP:29200" --pipeline=benchmark-only --report-file=report.md` +压测过程需要1-2个小时，部分压测结果如下： + +``` bash +------------------------------------------------------ + _______ __ _____ + / ____(_)___ ____ _/ / / ___/_________ ________ + / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \ + / __/ / / / / / /_/ / / ___/ / /__/ /_/ / / / __/ +/_/ /_/_/ /_/\__,_/_/ /____/\___/\____/_/ \___/ +------------------------------------------------------ + +| Lap | Metric | Task | Value | Unit | +|------:|-------------------------------------:|-------------:|------------:|--------:| +... +| All | Min Throughput | index-append | 16903.2 | docs/s | +| All | Median Throughput | index-append | 17624.4 | docs/s | +| All | Max Throughput | index-append | 19382.8 | docs/s | +| All | 50th percentile latency | index-append | 1865.74 | ms | +| All | 90th percentile latency | index-append | 3708.04 | ms | +| All | 99th percentile latency | index-append | 6379.49 | ms | +| All | 99.9th percentile latency | index-append | 8389.74 | ms | +| All | 99.99th percentile latency | index-append | 9612.84 | ms | +| All | 100th percentile latency | index-append | 9861.02 | ms | +| All | 50th percentile service time | index-append | 1865.74 | ms | +| All | 90th percentile service time | index-append | 3708.04 | ms | +| All | 99th percentile service time | index-append | 6379.49 | ms | +| All | 99.9th percentile service time | index-append | 8389.74 | ms | +| All | 99.99th percentile service time | index-append | 9612.84 | ms | +| All | 100th percentile service time | index-append | 9861.02 | ms | +| All | error rate | index-append | 0 | % | +| All | Min Throughput | default | 0.66 | ops/s | +| All | Median Throughput | default | 0.66 | ops/s | +| All | Max Throughput | default | 0.66 | ops/s | +| All | 50th percentile latency | default | 770131 | ms | +| All | 90th percentile latency | default | 825511 | ms | +| All | 99th percentile latency | default | 838030 | ms | +| All | 100th percentile latency | default | 839382 | ms | +| All | 50th percentile service time | default | 1539.4 | ms | +| All | 90th percentile service time | default | 1635.39 | ms | +| All | 99th percentile service time | default | 1728.02 | ms | +| All | 100th percentile service time | default | 1736.2 | ms | +| All | error rate | default | 0 | % | +... +``` + +从测试结果看：集群的吞吐可以（k8s es-client pod还可以扩展）；延迟略高一些（因为使用了nfs共享存储）；整体效果不错。 + +### 中文分词安装 + +安装 ik 插件即可，可以自定义已安装ik插件的es docker镜像：创建如下 Dockerfile + +``` bash +FROM jmgao1983/elasticsearch:6.4.0 + +RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install \ + --batch https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v6.4.0/elasticsearch-analysis-ik-6.4.0.zip \ + && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime +``` + +### 参考阅读 + +1. [Elasticsearch 入门教程](http://www.ruanyifeng.com/blog/2017/08/elasticsearch.html) +2. [Elasticsearch 压测方案之 esrally 简介](https://segmentfault.com/a/1190000011174694) diff --git a/docs/practice/java_war_app.md b/docs/practice/java_war_app.md new file mode 100644 index 0000000..89f7c7a --- /dev/null +++ b/docs/practice/java_war_app.md @@ -0,0 +1,159 @@ +# JAVA WAR 应用迁移 K8S 实践 + +初步思路是这样：应用代码与应用配置分离，应用代码打包成 docker 镜像存于内部 harbor 仓库，应用配置使用 configmap 挂载，这样不同的环境只需要修改 configmap 即可部署。 + +- 使用 maven 把 java 应用代码打包成 xxx.war +- 基于 tomcat 镜像和 xxx.war 做成应用 docker 镜像 +- 编写 k8s deployment 文件，在 pod 指定上述应用镜像，同时把应用配置做成 configmap 挂载到 pod 里 + +经过多次尝试部署发现问题：configmap配置是可以挂载上去，但是会把目录下其他的文件删掉，而且tomcat 目录 webapps/xxxxx/下其他目录也消失了。原来是因为 tomcat 容器完全启动完成后才会解压 war包，而 configmap 配置文件是一开始就挂载上去了，导致失败。 + +- 调整应用镜像打包过程：xxx.war 先解压后再进行应用镜像打包 + +## 应用 gitlab CI/CD 集成 + +- 在内部gitlab创建项目，上传应用java代码，同时在项目根目录下新加如下目录和文件，配置相应的 gitlab-runner 和 环境变量参数 + +``` bash +├── .app.yaml # k8s deployment 部署模板文件 +├── config.yaml # k8s configmap 配置模板文件 +├── dockerfiles +│   └── Dockerfile # Dockerfile 文件 +├── .gitlab-ci.yml # gitlab ci 配置文件 +└── .ns.yaml # k8s namespace 和 imagePullSecrets的配置文件 +``` +### gitlab-ci 文件摘要 + +``` bash +variables: + PROJECT_NS: '$CI_PROJECT_NAMESPACE-$CI_JOB_STAGE' + APP_NAME: '$CI_PROJECT_NAME-$CI_COMMIT_REF_SLUG' + +stages: + - package + - beta + +job_package: + stage: package + tags: + - package-shell + only: + - master + - /^feature-.*$/ + script: + - mvn clean install -Dmaven.test.skip=true + - unzip target/xxxx.war -d dockerfiles/project + - cd dockerfiles && docker build -t harbor.test.lo/project/$CI_PROJECT_NAME:$CI_PIPELINE_ID . + - docker login -u $HARBOR_USR -p $HARBOR_PWD harbor.test.lo + - docker push harbor.test.lo/project/$CI_PROJECT_NAME:$CI_PIPELINE_ID + - docker logout harbor.test.lo + +job_push_beta: + stage: beta + tags: + - beta-shell + only: + - master + - /^feature-.*$/ + when: manual + script: + # 替换beta环境的参数配置 + - sed -i "s/PROJECT_NS/$PROJECT_NS/g" config.yaml .app.yaml .ns.yaml + - sed -i "s/TemplateProject/$APP_NAME/g" config.yaml .app.yaml + - sed -i "s/DB_HOST/$BETA_DB_HOST/g" config.yaml + - sed -i "s/DB_PWD/$BETA_DB_PWD/g" config.yaml + - sed -i "s/APP_REP/$BETA_APP_REP/g" .app.yaml + - sed -i "s/ProjectImage/$CI_PROJECT_NAME:$CI_PIPELINE_ID/g" .app.yaml + # + - mkdir -p /opt/kube/$PROJECT_NS/$APP_NAME + - cp -f .ns.yaml config.yaml .app.yaml /opt/kube/$PROJECT_NS/$APP_NAME + - kubectl --kubeconfig=/etc/.beta/config apply -f .ns.yaml + - kubectl --kubeconfig=/etc/.beta/config apply -f config.yaml + - kubectl --kubeconfig=/etc/.beta/config apply -f .app.yaml + +# 生产部署与beta环境类同，这里省略 +``` + +### Dockerfile 编写 + +``` +FROM tomcat:8.5.33-jre8-alpine + +COPY . /usr/local/tomcat/webapps/ + +# 设置tomcat日志使用的时区 +RUN sed -i 's/^JAVA_OPTS=.*webresources\"$/JAVA_OPTS=\"$JAVA_OPTS -Djava.protocol.handler.pkgs=org.apache.catalina.webresources -Duser.timezone=GMT+08\"/g' /usr/local/tomcat/bin/catalina.sh +``` + +### k8s deployment 配置举例 + +``` +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: TemplateProject + namespace: PROJECT_NS +spec: + replicas: APP_REP + template: + metadata: + labels: + run: TemplateProject + spec: + containers: + - name: TemplateProject + image: harbor.test.lo/project/ProjectImage + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + volumeMounts: + - name: db-config + mountPath: "/usr/local/tomcat/webapps/project/xxxx/yyyy/config/datasource.properties" + subPath: datasource.properties + imagePullSecrets: + - name: projectkey1 + volumes: + - name: db-config + configMap: + name: TemplateProject-config + defaultMode: 0640 + items: + - path: datasource.properties + key: datasource.properties + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: TemplateProject + name: TemplateProject + namespace: PROJECT_NS +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: + run: TemplateProject + sessionAffinity: None +``` + +### k8s configmap 配置举例 + +``` +apiVersion: v1 +kind: ConfigMap +metadata: + name: TemplateProject-config + namespace: PROJECT_NS +data: + datasource.properties: | + dataSource.maxIdle = 5 + dataSource.maxActive = 41 + dataSource.driverClassName = com.mysql.jdbc.Driver + dataSource.url = jdbc:mysql://DB_HOST:8066/project?useUnicode=true&characterEncoding=utf-8 + dataSource.username = username + dataSource.password = DB_PWD +``` diff --git a/docs/practice/mariadb_cluster.md b/docs/practice/mariadb_cluster.md new file mode 100644 index 0000000..b5e333a --- /dev/null +++ b/docs/practice/mariadb_cluster.md @@ -0,0 +1,103 @@ +# Mariadb 数据库集群 + +Mariadb 是从 MySQL 衍生出来的开源关系型数据库，目前兼容 mysql 5.7 版本；它也非常流行，拥有 Google Facebook 等重要企业用户。本文档介绍使用 helm charts 方式安装 mariadb cluster，仅供实践交流使用。 + +## 前提条件 + +- 已部署 k8s 集群，参考[这里](../setup/quickStart.md) +- 已部署 helm，参考[这里](../guide/helm.md) +- 集群提供持久性存储，参考[这里](../setup/08-cluster-storage.md) + +这里演示使用 nfs 动态存储，编辑修改 nfs 存储部分参数 + +``` bash +$ vi roles/cluster-storage/defaults/main.yml +storage: + # nfs server 参数 + nfs: + enabled: "yes" # 启用 nfs + server: "172.16.3.86" # 设置 nfs 服务器地址 + server_path: "/data/nfs" # 设置共享目录 + storage_class: "nfs-db" # 定义 storage_class，后面pvc要调用这个 + provisioner_name: "nfs-provisioner-01" # 任意命名 + +# 配置完成，保存退出，运行下面命令 +$ ansible-playbook /etc/ansible/roles/cluster-storage/cluster-storage.yml +# 确认nfs provisioner pod +$ kubectl get pod --all-namespaces |grep nfs +kube-system nfs-provisioner-01-88694d78c-mrn7f 1/1 Running 0 6m +``` + +## mariadb charts 配置修改 + +按照惯例，直接把 chart 下载到本地，然后把配置复制 values.yaml 出来进行修改，这样方便以后整体更新 chart，安装实际使用需要修改配置文件 + +``` bash +$ cd /etc/ansible/manifests/mariadb-cluster +# 编辑 my-values.yaml 修改以下部分 + +service: + type: NodePort # 方便集群外部访问 + port: 3306 + nodePort: + master: 33306 # 设置主库的nodePort + slave: 33307 # 设置从库的nodePort + +rootUser: # 设置 root 密码 + password: test.c0m + forcePassword: true + +db: # 设置初始测试数据库 + user: hello + password: hello + name: hello + forcePassword: true + +replication: # 设置主从复制 + enabled: true + user: replicator + password: R4%forep11CAT0r + forcePassword: true + +master: + affinity: {} + antiAffinity: soft + tolerations: [] + persistence: + enabled: true # 启用持久化存储 + mountPath: /bitnami/mariadb + storageClass: "nfs-db" # 设置使用 nfs-db 存储类 + annotations: {} + accessModes: + - ReadWriteOnce + size: 5Gi # 设置存储容量 + +slave: + replicas: 1 + affinity: {} + antiAffinity: soft + tolerations: [] + persistence: + enabled: false # 从库这里没有启用持久性存储 +``` + +## 安装 + +使用 helm 安装 + +``` bash +$ cd /etc/ansible/manifests/mariadb-cluster +$ helm install --name mariadb --namespace default -f my-values.yaml ./mariadb +``` + +## 验证 + +``` bash +$ kubectl get pod,svc | grep mariadb +pod/mariadb-mariadb-master-0 1/1 Running 0 27m +pod/mariadb-mariadb-slave-0 1/1 Running 0 29m + +service/mariadb NodePort 10.68.170.168 3306:33306/TCP 29m +service/mariadb-mariadb-slave NodePort 10.68.151.95 3306:33307/TCP 29m +``` + diff --git a/docs/release-notes/kubeasz-0.1.0.md b/docs/release-notes/kubeasz-0.1.0.md new file mode 100644 index 0000000..01e6fa1 --- /dev/null +++ b/docs/release-notes/kubeasz-0.1.0.md @@ -0,0 +1,33 @@ +## kubeasz-0.1.0 发布说明 + +`kubeasz`项目第一个独立版本发布，使用`ansible playbook`自动化安装k8s集群（目前支持v1.8/v1.9/v1.10）和主要插件，方便部署和灵活配置集群； + +CHANGELOG: +- 组件更新： + - kubernetes v1.10.4, v1.9.8, v1.8.12 + - etcd v3.3.6 +- 安全更新： + - 修复kubelet匿名访问漏洞（感谢 cqspirit #192 提醒） +- 功能更新： + - 增加helm安全部署及说明 + - 增加prometheus部署及说明 + - 增加jenkins部署及说明（感谢 lusyoe #208 ） +- 脚本更新： + - 精简 inventory（/etc/ansible/hosts）配置项 + - 移动calico/flannel配置至对应的roles/defaults/main.yml + - 取消变量NODE_IP，使用内置变量inventory_hostname代替 + - 取消lb组变量设置，自动完成 + - 取消etcd相关集群变量设置，自动完成 + - 增加集群版本K8S_VER变量，为兼容k8s v1.8安装 + - 增加修改AIO部署的系统IP的脚本和说明(docs/op/change_ip_allinone.md) + - 增加设置node角色 + - 修改OS安全加固脚本为可选安装 +- 其他： + - 修复calico-controller多网卡问题 + - 修改manifests/apiserver参数兼容k8s v1.8 + - 简化新增master/node节点步骤 + - 优化ansible配置参数 + - 更新 harbor 1.5.1及文档修复（感谢 lusyoe #224 ） + - 更新 kube-dns 1.14.10 + - 丰富dashboard文档（ #182 ） + - 修复selinux关闭（ #194 ） diff --git a/docs/release-notes/kubeasz-0.2.0.md b/docs/release-notes/kubeasz-0.2.0.md new file mode 100644 index 0000000..e8d5533 --- /dev/null +++ b/docs/release-notes/kubeasz-0.2.0.md @@ -0,0 +1,19 @@ +## kubeasz-0.2.0 发布说明 + +CHANGELOG: +- 组件更新： + - 增加新网络插件 kube-router，可在ansible hosts配置`CLUSTER_NETWORK="kube-router"` +- 功能更新： + - 增加IPVS/LVS服务代理模式，比默认的kube-proxy服务代理更高效；在选择kube-router网络插件时配置`SERVICE_PROXY="IPVS"` + - 增加部署metrics-server，以替代heapster 提供metrics API + - 增加自动集成安装kube-dns/dashboard等组件，可在`roles/cluster-addon/defaults/main.yml`配置 +- 脚本更新： + - 增加删除单个节点脚本，docs/op/del_one_node.md + - 增加等待网络插件正常运行 + - Bug fix: 更新99.clean.yml清理脚本，解决集群重装后cni地址分配问题 kubernetes #57280 + - Bug fix: 从0.1.0版本升级时，kube-apiserver服务启动失败问题 +- 其他： + - 修改部分镜像拉取策略统一为：`imagePullPolicy: IfNotPresent` + - 新增metrics-server、cluster-addon文档 + - 更新kube-router相关文档 + - 更新集群升级说明文档 docs/op/upgrade.md diff --git a/docs/release-notes/kubeasz-0.2.1.md b/docs/release-notes/kubeasz-0.2.1.md new file mode 100644 index 0000000..b2ac198 --- /dev/null +++ b/docs/release-notes/kubeasz-0.2.1.md @@ -0,0 +1,18 @@ +## kubeasz-0.2.1 发布说明 + +CHANGELOG: +如果服务器能够使用内部yum源/apt源，但是无法访问公网情况下，请下载离线docker镜像完成集群安装：从百度云盘把`basic_images_kubeasz_x.y.tar.gz` 下载解压到项目`down`目录即可 +- 组件更新： + - 更新 coredns版本1.1.3 +- 功能更新： + - 集成网络插件（可选）使用离线docker镜像安装 + - 集成其他插件（可选）使用离线docker镜像安装 + - 增加切换集群网络插件的脚本 +- 文档更新： + - [快速指南](https://github.com/gjmzj/kubeasz/blob/master/docs/setup/quickStart.md) + - [安装规划](https://github.com/gjmzj/kubeasz/blob/master/docs/setup/00-planning_and_overall_intro.md) + - [切换网络](https://github.com/gjmzj/kubeasz/blob/master/docs/op/clean_k8s_network.md) +- 其他： + - Bug fix: 清理集群时可能出现`Device or resource busy: '/var/run/docker/netns/xxxxxxx'`的错误，可手动umount后重新清理集群 + - Bug fix: #239 harbor调整安装解压工具, 适配多系统 (#240) + diff --git a/docs/release-notes/kubeasz-0.2.2.md b/docs/release-notes/kubeasz-0.2.2.md new file mode 100644 index 0000000..2714b74 --- /dev/null +++ b/docs/release-notes/kubeasz-0.2.2.md @@ -0,0 +1,22 @@ +## kubeasz-0.2.2 发布说明 + +CHANGELOG: +- 组件更新： + - k8s v1.11.0 + - etcd v3.3.8 + - docker 18.03.1-ce +- 功能更新： + - 更新使用ipvs 配置及[说明文档](https://github.com/gjmzj/kubeasz/blob/master/docs/guide/ipvs.md) + - 更新lb节点keepalived使用单播发送vrrp报文，预期兼容公有云上自建LB（待测试） + - 废弃原 ansible hosts 中变量SERVICE_PROXY + - 更新haproxy负载均衡算法配置 +- 其他修复： + - fix 变更集群网络的脚本和[文档](https://github.com/gjmzj/kubeasz/blob/master/docs/op/change_k8s_network.md) + - fix 脚本99.clean.yml清理环境变量 + - fix metrics-server允许的client cert问题 + - fix #242: 添加CA有效期参数，设定CA有效期为15年(131400h) (#245) + - fix helm安装出现Error: transport is closing (#248) + - fix harbor点击tag界面出现\"发生未知错误,请稍后再试" (#250) + - fix 脚本99.clean.yml清理 services softlink (#253) + - fix kube-apiserver-v1.8 使用真实数量的 apiserver-count (#254) + - fix 清理ipvs产生的网络接口 diff --git a/docs/release-notes/kubeasz-0.3.0.md b/docs/release-notes/kubeasz-0.3.0.md new file mode 100644 index 0000000..af4ba52 --- /dev/null +++ b/docs/release-notes/kubeasz-0.3.0.md @@ -0,0 +1,26 @@ +## kubeasz-0.3.0 发布说明 + +CHANGELOG: +- 组件更新： + - k8s: v1.11.2/v1.10.6/v1.9.10/v1.8.15 + - calico: v3.1.3 + - kube-router: v0.2.0-beta.9 +- 功能更新： + - **增加集群备份与恢复** 功能与[说明](https://github.com/gjmzj/kubeasz/blob/master/docs/op/cluster_restore.md) + - **增加cilium网络插件** ，文档待更新 + - **增加cluster-storage角色** 与[文档说明](https://github.com/gjmzj/kubeasz/blob/master/docs/setup/08-cluster-storage.md) + - 增加阿里云NAS存储支持 + - 增加集群个性化[配置说明](https://github.com/gjmzj/kubeasz/blob/master/docs/setup/config_guide.md)与生成脚本`tools/init_vars.yml` + - 支持deploy节点与ansible执行节点分离，为一份代码创建多个集群准备 +- 其他： + - 更新 jenkins and plugins (#258) + - 重写 nfs动态存储脚本与文档 + - 优化 cluster-addon 安装脚本 + - 增加 docker 配置文件 + - 更新 offline images 0.3 + - 增加 batch/v2alpha支持 + - 移动 DNS yaml文件至 /opt/kube/kube-system + - fix 多主集群下change_k8s_network时vip丢失问题 + - fix 禁止节点使用系统swap + - fix 解压后的harbor安装文件没有执行权限问题 + - fix Ubuntu 18.04无法安装haproxy、keepalived问题 diff --git a/docs/release-notes/kubeasz-0.3.1.md b/docs/release-notes/kubeasz-0.3.1.md new file mode 100644 index 0000000..0bd1154 --- /dev/null +++ b/docs/release-notes/kubeasz-0.3.1.md @@ -0,0 +1,30 @@ +## kubeasz-0.3.1 发布说明 + +CHANGELOG: +- 组件更新： + - k8s: v1.11.3, v1.10.7 + - kube-router: v0.2.0 + - dashboard: v1.10.0 + - docker: 17.03.2-ce (选择k8s官方测试稳定的版本) +- 集群安装： + - **增加集群时间同步服务chrony** [说明](https://github.com/gjmzj/kubeasz/blob/master/docs/guide/chrony.md) + - **取消 Node节点 Bootstrap机制**，安装流程更稳定，配置更精简 + - 简化 ansible host 文件配置，移除etcd、harbor 相关变量 + - 拆分 prepare 阶段的安装脚本，增加设置系统 ulimit + - 增加多lb节点（多于2节点）配置支持 (#286) + - 增加可选配置lb 节点负载转发ingress controller NodePort service的功能 + - 自定义 kubelet docker 存储目录 (#305) + - 增加变量配置支持多网卡情况时安装 flannel calico +- 文档更新： + - 更新 kubeasz 公有云安装文档 https://github.com/gjmzj/kubeasz/blob/master/docs/setup/kubeasz_on_public_cloud.md + - 更新 java war应用部署实践 https://github.com/gjmzj/kubeasz/blob/master/docs/practice/java_war_app.md + - 更新 cilium 文档，翻译官方 cilium 安全策略例子（deathstar/starwar） + - 更新 harbor kubedns README 文档 + - 更新集群安装部分文档 +- 其他： + - 修复 calicoctl 配置，修复calico/node跑在LB 主节点时使用`vip`作为`bgp peer`地址问题 + - 修复 jq安装错误，补充ipset和ipvsadm安装 + - 修复清除单节点脚本 tools/clean_one_node.yml + - 修复消除离线镜像不存在时安装的错误提示信息 + - 修复多节点（超过2节点时）lb 备节点 router_id重复问题 + - 锁定jenkins镜像tag、升级插件版本以及锁定安全插件 (#315) diff --git a/docs/release-notes/kubeasz-0.4.0.md b/docs/release-notes/kubeasz-0.4.0.md new file mode 100644 index 0000000..95c2fe2 --- /dev/null +++ b/docs/release-notes/kubeasz-0.4.0.md @@ -0,0 +1,29 @@ +## kubeasz-0.4.0 发布说明 + +CHANGELOG: +- 组件更新： + - k8s: v1.12.1, v1.10.8, v1.9.11 [注意 v1.12.1 kubelet日志bug](https://github.com/kubernetes/kubernetes/issues/69503) + - docker: 18.06.1-ce (选择k8s官方测试稳定的版本) + - metrics-server: v0.3.1 + - coredns: 1.2.2, kube-dns 1.14.13 + - heapster v1.5.4 + - traefik 1.7.2 +- 集群安装： + - **更新 kubelet使用 webhook方式认证/授权** ，提高集群安全性 + - 调整安装步骤中 kubectl 命令的执行以兼容公有云部署 + - 调整部分安装步骤以兼容`ansible`执行节点与`deploy`节点分离 + - 更新节点的安全加固脚本[ansible-os-hardening 5.0.0](https://github.com/dev-sec/ansible-os-hardening) +- 文档更新： + - 新增`elasticsearch`集群[部署实践](https://github.com/gjmzj/kubeasz/blob/master/docs/practice/es_cluster.md) + - 更新[kubeasz 公有云安装文档](https://github.com/gjmzj/kubeasz/blob/master/docs/setup/kubeasz_on_public_cloud.md) + - 调整集群安装步骤文档目录及修改使用英文文件名 + - 修改部分脚本内部注释为英文 +- 其他： + - 升级 promethus chart 7.1.4，grafana chart 1.16.0 + - 升级 jenkins 安全插件和 k8s 插件版本 (#325) + - 修复 新增 master 节点时报变量未定义错误 + - 修复 ipvs 模式下网络组件偶尔连不上`kubernetes svc`的错误 + - 修复 Ansible 2.7 环境下 yum/apt 安装多个软件包的 DEPRECATION WARNING (#334) + - 修复 chrony 与 ntp 共存冲突问题 (#341) + - 修复 CentOS 下使用 ipvs 模式需依赖 conntrack-tools 软件包 + - 修复 tools/change_k8s_network.yml 脚本 diff --git a/docs/release-notes/kubeasz-0.5.0.md b/docs/release-notes/kubeasz-0.5.0.md new file mode 100644 index 0000000..2ca19cd --- /dev/null +++ b/docs/release-notes/kubeasz-0.5.0.md @@ -0,0 +1,29 @@ +## kubeasz-0.5.0 发布说明 + +CHANGELOG: +- 组件更新： + - k8s: v1.12.3, v1.11.5, v1.10.11 + - calico v3.2.4 + - helm v2.11.0 + - traefik 1.7.4 +- 集群安装： + - 更新集群升级脚本和[文档](https://github.com/gjmzj/kubeasz/blob/master/docs/op/upgrade.md)，关注[安全漏洞](https://mp.weixin.qq.com/s/Q8XngAr5RuL_irRscbVbKw) + - 集成 metallb 作为自有硬件 k8s 集群的 LoadBalancer 实现 + - 支持[修改 APISERVER 证书](https://github.com/gjmzj/kubeasz/blob/master/docs/op/ch_apiserver_cert.md) + - 增加 ingress nodeport 负载转发的脚本与[文档](https://github.com/gjmzj/kubeasz/blob/master/docs/op/loadballance_ingress_nodeport.md) + - 增加 https ingress 配置和[文档](https://github.com/gjmzj/kubeasz/blob/master/docs/guide/ingress-tls.md) + - 增加 kubectl 只读访问权限配置和[文档](https://github.com/gjmzj/kubeasz/blob/master/docs/op/readonly_kubectl.md) + - 增加 apiserver 配置支持 istio sidecar自动注入webhook (#375) + - 初始化集群节点设置 net.netfilter.nf_conntrack_max=1000000 + - 取消多主集群LB_IF参数设置，自动生成以避免人为配置疏忽 +- 文档更新： + - 更新[kubeasz 公有云安装文档](https://github.com/gjmzj/kubeasz/blob/master/docs/setup/kubeasz_on_public_cloud.md) + - 更新[metallb 文档](https://github.com/gjmzj/kubeasz/blob/master/docs/guide/metallb.md) + - 更新[dashboard 文档](https://github.com/gjmzj/kubeasz/blob/master/docs/guide/dashboard.md)，支持只读权限设置 + - 新增istio安装说明 +- 其他： + - 修复内核4.19加载nf_conntrack (#366) + - 修复 calico controller 中 NodePorts 的自动配置 + - 取消 helms 别名设置 + - 升级jenkins-lts版本和插件版本 (#358) + - 修复阿里云nas动态pv脚本 diff --git a/docs/release-notes/kubeasz-0.5.1.md b/docs/release-notes/kubeasz-0.5.1.md new file mode 100644 index 0000000..928175c --- /dev/null +++ b/docs/release-notes/kubeasz-0.5.1.md @@ -0,0 +1,24 @@ +## kubeasz-0.5.1 发布说明 + +CHANGELOG: +- 组件更新： + - k8s: v1.13.2, v1.12.4, v1.11.6, v1.10.12 + - calico v3.3.2 + - coredns 1.2.6 +- 集群安装： + - 更新 calico 3.3.2，并保留3.2.4可选 + - 修复特定环境下lb节点变量LB_IF自动设置错误 + - 移除 kube-node csr 请求批准部分（PR #399） + - 添加支持 RedHat （PR #431） + - 修改 docker 存储的目录设置（PR #436） + - 更新 kube-schedule 监听参数 （PR #440） + - 安装流程增加等待 ETCD 同步完成再返回成功（PR #420） + - 增加 pod-infra-container 可选择配置 + - 增加 nginx-ingress manifests +- 文档更新： + - **增加 [calico 设置route reflector文档](https://github.com/gjmzj/kubeasz/blob/master/docs/setup/network-plugin/calico-bgp-rr.md)**，大规模k8s集群使用calico网络必读 + - 部分文档更新优化，部分文档中内部链接修复（PR #429） + - 增加 dashboard ingress [配置示例](https://github.com/gjmzj/kubeasz/blob/master/docs/guide/ingress-tls.md#%E9%85%8D%E7%BD%AE-dashboard-ingress) +- 其他： + - 添加 helm tls 环境变量（PR #398） + - 修复 dashboard ingress 配置（issue #403） diff --git a/docs/release-notes/kubeasz-0.6.0.md b/docs/release-notes/kubeasz-0.6.0.md new file mode 100644 index 0000000..66e29b8 --- /dev/null +++ b/docs/release-notes/kubeasz-0.6.0.md @@ -0,0 +1,36 @@ +## kubeasz-0.6.0 发布说明 + +- Note: 本次为 kubeasz-0.x 最后一次版本发布，它将被并入 release-0 分支，停止主要更新，仅做 bug 修复版本；后续 master 分支将开始 kubeasz-1.x 版本发布。 +- Action Required: 本次更新修改 ansible hosts 文件，如需要更新已有项目使用，请按照 example 目录中的对应例子修改`/etc/ansible/hosts`文件。 + +CHANGELOG: +- 组件更新： + - k8s: v1.13.3 + - calico v3.4.1 + - flannel v0.11.0-amd64 + - docker 18.09.2 + - harbor 1.6.3 + - helm/tiller: v2.12.3 +- 集群安装： + - **增加添加/删除 etcd 节点**脚本和[文档](https://github.com/gjmzj/kubeasz/blob/master/docs/op/op-etcd.md) + - **增加可选配置附加负载均衡节点（ex-lb）**，可用于负载均衡 NodePort 方式暴露的服务 + - 更新删除节点脚本和[文档](https://github.com/gjmzj/kubeasz/blob/master/docs/op/del_one_node.md) + - 优化增加 node 和增加 master 节点流程 + - 更新 harbor 安装流程和文档 + - 优化 prepare tasks，避免把证书和 kubeconfig 分发到不需要的节点 + - 更新 prometheus 告警发送钉钉配置和[文档](https://github.com/gjmzj/kubeasz/blob/master/docs/guide/prometheus.md#%E5%8F%AF%E9%80%89-%E9%85%8D%E7%BD%AE%E9%92%89%E9%92%89%E5%91%8A%E8%AD%A6) + - 增加使用 helm 部署 mariadb 集群和文档 + - 增加 k8s 官方 mysql 集群示意配置 + - 增加使用 helm 部署 redis-ha 集群 + - 增加开机启动 k8s 相关内核模块配置 + - 更新 calico 3.4.1，并保留版本 3.3.x/3.2.x 可选 +- 文档更新： + - **增加 gitlab-ci 文档**, https://github.com/gjmzj/kubeasz/blob/master/docs/guide/gitlab/readme.md + - 部分文档更新（helm/dns/chrony） +- 其他： + - 修复为兼容k8s版本 <= 1.11，revert PR #440 + - 修复清除iptables规则时无法连接节点（PR #453 by PowerDos） + - 添加开启docker远程API选项（默认关闭）（PR #444 by lusyoe） + - 修复 calico 3.3.x rbac 配置（PR #447 by sunshanpeng） + - 增加 coredns 和 calico 的 metrics 监控选项（PR #447 by sunshanpeng） + - 添加 helm 离线安装方法说明（doc/guide/helm.md）（PR #443 by j4ckzh0u） diff --git a/docs/release-notes/kubeasz-1.0.0rc1 b/docs/release-notes/kubeasz-1.0.0rc1 new file mode 100644 index 0000000..7a3ec4a --- /dev/null +++ b/docs/release-notes/kubeasz-1.0.0rc1 @@ -0,0 +1,26 @@ +## kubeasz-1.0.0rc1 发布说明 + +- Note: kubeasz-1.x 第一个版本预发布，原 master 已并入 release-0 分支，停止主要更新，仅做 bug 修复版本；后续 master 分支将开始 kubeasz-1.x 版本发布。 +- Action Required: 本次更新修改 ansible hosts 文件，请按照 example 目录中的对应例子修改`/etc/ansible/hosts`文件, 确保 ansible hosts 文件中主机组的顺序与例子一致。 + +CHANGELOG: +- 组件更新： + - k8s: v1.13.4 + - cilium v1.4.1 +- 集群安装： + - **引入[easzctl](https://github.com/gjmzj/kubeasz/blob/master/tools/easzctl)命令行工具**，后续它将作为推荐的集群常规管理工具，包括多集群管理(to do) + - **新增 docker 运行安装 kubeasz**，请参考文档 https://github.com/gjmzj/kubeasz/blob/master/docs/setup/docker_kubeasz.md + - 优化 example hosts 配置，废弃 new-node/new-master/new-etcd 主机组，废弃变量K8S_VER，改为自动识别 + - 集成以下集群操作至 easzctl 命令行 + - [添加 master](https://github.com/gjmzj/kubeasz/blob/master/docs/op/AddMaster.md) + - [添加 node](https://github.com/gjmzj/kubeasz/blob/master/docs/op/AddNode.md) + - [添加 etcd](https://github.com/gjmzj/kubeasz/blob/master/docs/op/op-etcd.md) + - [删除 etcd](https://github.com/gjmzj/kubeasz/blob/master/docs/op/op-etcd.md) + - [删除节点](https://github.com/gjmzj/kubeasz/blob/master/docs/op/clean_one_node.md) + - [快速创建 aio 集群]() + - 修改安装时生成随机 basic auth 密码 + - 修改优化部分安装脚本以兼容 docker 运行 kubeasz + - update cilium v1.4.1，更新 cilium 文档(to do) + - 增加启动 kubeasz 容器的脚本 tools/kubeasz-docker +- 其他： + - 修复兼容 docker 18.09.x 版本安装 diff --git a/docs/setup/00-planning_and_overall_intro.md b/docs/setup/00-planning_and_overall_intro.md new file mode 100644 index 0000000..f986eec --- /dev/null +++ b/docs/setup/00-planning_and_overall_intro.md @@ -0,0 +1,151 @@ +## 00-集群规划和基础参数设定 + +多节点高可用集群部署步骤与[AllinOne部署](quickStart.md)基本一致，增加LB 负载均衡部署步骤。 + +- 注意1：请确保各节点时区设置一致、时间同步。 如果你的环境没有提供NTP 时间同步，推荐集成安装[chrony](../guide/chrony.md) +- 注意2：如果需要在公有云上创建多主多节点集群，请结合阅读[在公有云上部署 kubeasz](kubeasz_on_public_cloud.md) + +## 高可用集群所需节点配置如下 + +|角色|数量|描述| +|:-|:-|:-| +|deploy节点|1|运行这份 ansible 脚本的节点| +|etcd节点|3|注意etcd集群必须是1,3,5,7...奇数个节点| +|master节点|2|需要额外规划一个master VIP(虚地址)，可根据需要提升机器配置或增加节点数| +|lb节点|2|负载均衡节点两个，安装 haproxy+keepalived| +|node节点|3|运行应用负载的节点，可根据需要提升机器配置或增加节点数| + +项目预定义了4个例子，请修改后完成适合你的集群规划，生产环境建议一个节点只是一个角色。 + ++ [单节点](../../example/hosts.allinone.example) ++ [单主多节点](../../example/hosts.s-master.example) ++ [多主多节点](../../example/hosts.m-masters.example) ++ [在公有云上部署](../../example/hosts.cloud.example) + +## 部署步骤 + +按照[多主多节点](../../example/hosts.m-masters.example)示例的节点配置，准备4台虚机，搭建一个多主高可用集群。 + +### 1.基础系统配置 + ++ 推荐内存2G/硬盘30G以上 ++ 最小化安装`Ubuntu 16.04 server`或者`CentOS 7 Minimal` ++ 配置基础网络、更新源、SSH登陆等 + +### 2.在每个节点安装依赖工具 + +Ubuntu 16.04 请执行以下脚本: + +``` bash +# 文档中脚本默认均以root用户执行 +apt-get update && apt-get upgrade -y && apt-get dist-upgrade -y +# 安装python2 +apt-get install python2.7 +# Ubuntu16.04可能需要配置以下软连接 +ln -s /usr/bin/python2.7 /usr/bin/python +``` +CentOS 7 请执行以下脚本： + +``` bash +# 文档中脚本默认均以root用户执行 +# 安装 epel 源并更新 +yum install epel-release -y +yum update +# 安装python +yum install python -y +``` +### 3.在deploy节点安装及准备ansible + +- pip 安装 ansible（如果 Ubuntu pip报错，请看[附录](00-planning_and_overall_intro.md#Appendix)） + +``` bash +# Ubuntu 16.04 +apt-get install git python-pip -y +# CentOS 7 +yum install git python-pip -y +# pip安装ansible(国内如果安装太慢可以直接用pip阿里云加速) +#pip install pip --upgrade +#pip install ansible +pip install pip --upgrade -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +pip install --no-cache-dir ansible -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +``` + +- 在deploy节点配置免密码登陆 + +``` bash +ssh-keygen -t rsa -b 2048 回车 回车 回车 +ssh-copy-id $IPs #$IPs为所有节点地址包括自身，按照提示输入yes 和root密码 +``` +### 4.在deploy节点编排k8s安装 + +- 4.1 下载项目源码 + +``` bash +# 方式一：使用git clone +git clone https://github.com/gjmzj/kubeasz.git +mkdir -p /etc/ansible +mv kubeasz/* /etc/ansible +# 方式二：从发布页面 https://github.com/gjmzj/kubeasz/releases 下载源码解压到同样目录 +``` +- 4.2a 下载二进制文件 +请从分享的[百度云链接](https://pan.baidu.com/s/1c4RFaA)，下载解压到/etc/ansible/bin目录，如果你有合适网络环境也可以按照/down/download.sh自行从官网下载各种tar包 + +``` bash +tar zxvf k8s.1-9-8.tar.gz # 以安装k8s v1.9.8为例 +mv bin/* /etc/ansible/bin +``` +- 4.2b [可选]下载离线docker镜像 +服务器使用内部yum源/apt源，但是无法访问公网情况下，请下载离线docker镜像完成集群安装；从百度云盘把`basic_images_kubeasz_x.y.tar.gz` 下载解压到`/etc/ansible/down` 目录 + +``` bash +tar zxvf basic_images_kubeasz_0.2.tar.gz -C /etc/ansible/down +``` +- 4.3 配置集群参数 + - 4.3.1 必要配置：`cd /etc/ansible && cp example/hosts.m-masters.example hosts`, 然后实际情况修改此hosts文件 + - 4.3.2 可选配置，初次使用可以不做修改，详见[配置指南](config_guide.md) + - 4.3.3 验证ansible 安装：`ansible all -m ping` 正常能看到节点返回 SUCCESS + +- 4.4 开始安装 +如果你对集群安装流程不熟悉，请阅读项目首页 **安装步骤** 讲解后分步安装，并对 **每步都进行验证** + +``` bash +# 分步安装 +ansible-playbook 01.prepare.yml +ansible-playbook 02.etcd.yml +ansible-playbook 03.docker.yml +ansible-playbook 04.kube-master.yml +ansible-playbook 05.kube-node.yml +ansible-playbook 06.network.yml +ansible-playbook 07.cluster-addon.yml +# 一步安装 +#ansible-playbook 90.setup.yml +``` + ++ [可选]对集群所有节点进行操作系统层面的安全加固 `ansible-playbook roles/os-harden/os-harden.yml`，详情请参考[os-harden项目](https://github.com/dev-sec/ansible-os-hardening) + +## Appendix + +- Ubuntu 1604 安装 ansible 如果出现以下错误 + +``` bash +Traceback (most recent call last): + File "/usr/bin/pip", line 9, in + from pip import main +ImportError: cannot import name main +``` +将`/usr/bin/pip`做以下修改即可 + +``` bash +#原代码 +from pip import main +if __name__ == '__main__': + sys.exit(main()) + +#修改后 +from pip import __main__ +if __name__ == '__main__': + sys.exit(__main__._main()) +``` + + +[后一篇](01-CA_and_prerequisite.md) diff --git a/docs/setup/01-CA_and_prerequisite.md b/docs/setup/01-CA_and_prerequisite.md new file mode 100644 index 0000000..909da08 --- /dev/null +++ b/docs/setup/01-CA_and_prerequisite.md @@ -0,0 +1,363 @@ +# 01-创建证书和环境配置 + +本步骤[01.prepare.yml](../../01.prepare.yml)主要完成: + +- chrony role: 集群节点时间同步[可选] +- deploy role: 创建CA证书、kubeconfig、kube-proxy.kubeconfig +- prepare role: 分发CA证书、kubectl客户端安装、环境配置 +- lb role: 安装负载均衡[可选] + +## deploy 角色 + +请在另外窗口打开[roles/deploy/tasks/main.yml](../../roles/deploy/tasks/main.yml) 文件，对照看以下讲解内容。 + +### 创建 CA 证书和秘钥 +``` bash +roles/deploy/ +├── tasks +│   └── main.yml +└── templates + ├── admin-csr.json.j2 # kubectl客户端使用的证书请求模板 + ├── ca-config.json.j2 # ca 配置文件模板 + ├── ca-csr.json.j2 # ca 证书签名请求模板 + ├── kubedns.yaml.j2 + └── kube-proxy-csr.json.j2 # kube-proxy使用的证书请求模板 +``` +kubernetes 系统各组件需要使用 TLS 证书对通信进行加密，使用 CloudFlare 的 PKI 工具集生成自签名的 CA 证书，用来签名后续创建的其它 TLS 证书。[参考阅读](https://coreos.com/os/docs/latest/generate-self-signed-certificates.html) + +根据认证对象可以将证书分成三类：服务器证书`server cert`，客户端证书`client cert`，对等证书`peer cert`(表示既是`server cert`又是`client cert`)，在kubernetes 集群中需要的证书种类如下： + ++ `etcd` 节点需要标识自己服务的`server cert`，也需要`client cert`与`etcd`集群其他节点交互，当然可以分别指定2个证书，也可以使用一个对等证书 ++ `master` 节点需要标识 apiserver服务的`server cert`，也需要`client cert`连接`etcd`集群，这里也使用一个对等证书 ++ `kubectl` `calico` `kube-proxy` 只需要`client cert`，因此证书请求中 `hosts` 字段可以为空 ++ `kubelet` 证书比较特殊，不是手动生成，它由node节点`TLS BootStrap` 向`apiserver`请求，由`master`节点的`controller-manager` 自动签发，包含一个`client cert` 和一个`server cert` + +整个集群要使用统一的CA 证书，只需要在 deploy 节点创建，然后分发给其他节点；为了保证安装的幂等性，如果已经存在CA 证书，就跳过创建CA 步骤 + +#### 创建 CA 配置文件 [ca-config.json.j2](../../roles/deploy/templates/ca-config.json.j2) +``` bash +{ + "signing": { + "default": { + "expiry": "87600h" + }, + "profiles": { + "kubernetes": { + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ], + "expiry": "87600h" + } + } + } +} +``` ++ `signing`：表示该证书可用于签名其它证书；生成的 ca.pem 证书中 `CA=TRUE`； ++ `server auth`：表示可以用该 CA 对 server 提供的证书进行验证； ++ `client auth`：表示可以用该 CA 对 client 提供的证书进行验证； ++ `profile kubernetes` 包含了`server auth`和`client auth`，所以可以签发三种不同类型证书； + +#### 创建 CA 证书签名请求 [ca-csr.json.j2](../../roles/deploy/templates/ca-csr.json.j2) +``` bash +{ + "CN": "kubernetes", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "CN", + "ST": "HangZhou", + "L": "XS", + "O": "k8s", + "OU": "System" + } + ], + "ca": { + "expiry": "876000h" + } +} +``` + +#### 生成CA 证书和私钥 +``` bash +cfssl gencert -initca ca-csr.json | cfssljson -bare ca +``` + +### 生成 kubeconfig 配置文件 + +kubectl使用~/.kube/config 配置文件与kube-apiserver进行交互，且拥有管理 K8S集群的完全权限， + +准备kubectl使用的admin 证书签名请求 [admin-csr.json.j2](../../roles/deploy/templates/admin-csr.json.j2) + +``` bash +{ + "CN": "admin", + "hosts": [], + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "CN", + "ST": "HangZhou", + "L": "XS", + "O": "system:masters", + "OU": "System" + } + ] +} + +``` ++ kubectl 使用客户端证书可以不指定hosts 字段 ++ 证书请求中 `O` 指定该证书的 Group 为 `system:masters`，而 `RBAC` 预定义的 `ClusterRoleBinding` 将 Group `system:masters` 与 ClusterRole `cluster-admin` 绑定，这就赋予了kubectl**所有集群权限** + +``` bash +$ kubectl describe clusterrolebinding cluster-admin +Name: cluster-admin +Labels: kubernetes.io/bootstrapping=rbac-defaults +Annotations: rbac.authorization.kubernetes.io/autoupdate=true +Role: + Kind: ClusterRole + Name: cluster-admin +Subjects: + Kind Name Namespace + ---- ---- --------- + Group system:masters +``` + +#### 生成 cluster-admin 用户证书 + +``` +cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin +``` + +#### 生成 ~/.kube/config 配置文件 + +使用`kubectl config` 生成kubeconfig 自动保存到 ~/.kube/config，生成后 `cat ~/.kube/config`可以验证配置文件包含 kube-apiserver 地址、证书、用户名等信息。 + +``` +kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=127.0.0.1:8443 +kubectl config set-credentials admin --client-certificate=admin.pem --embed-certs=true --client-key=admin-key.pem +kubectl config set-context kubernetes --cluster=kubernetes --user=admin +kubectl config use-context kubernetes +``` + +### 生成 kube-proxy.kubeconfig 配置文件 + +创建 kube-proxy 证书请求 + +``` bash +{ + "CN": "system:kube-proxy", + "hosts": [], + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "CN", + "ST": "HangZhou", + "L": "XS", + "O": "k8s", + "OU": "System" + } + ] +} +``` ++ kube-proxy 使用客户端证书可以不指定hosts 字段 ++ CN 指定该证书的 User 为 system:kube-proxy，预定义的 ClusterRoleBinding system:node-proxier 将User system:kube-proxy 与 Role system:node-proxier 绑定，授予了调用 kube-apiserver Proxy 相关 API 的权限； + +``` bash +$ kubectl describe clusterrolebinding system:node-proxier +Name: system:node-proxier +Labels: kubernetes.io/bootstrapping=rbac-defaults +Annotations: rbac.authorization.kubernetes.io/autoupdate=true +Role: + Kind: ClusterRole + Name: system:node-proxier +Subjects: + Kind Name Namespace + ---- ---- --------- + User system:kube-proxy +``` + +#### 生成 system:kube-proxy 用户证书 + +``` +cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy +``` + +#### 生成 kube-proxy.kubeconfig + +使用`kubectl config` 生成kubeconfig 自动保存到 kube-proxy.kubeconfig + +``` +kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=127.0.0.1:8443 --kubeconfig=kube-proxy.kubeconfig +kubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --embed-certs=true --client-key=kube-proxy-key.pem --kubeconfig=kube-proxy.kubeconfig +kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig +kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig +``` + +## prepare 角色 + +``` bash +roles/prepare/ +├── files +│   ├── 95-k8s-sysctl.conf +└── tasks + └── main.yml +``` +请在另外窗口打开[roles/prepare/tasks/main.yml](../../roles/prepare/tasks/main.yml) 文件，比较简单直观 + +1. 首先创建一些基础文件目录 +1. 修改环境变量，把{{ bin_dir }} 添加到$PATH，需要重新登陆 shell生效 +1. 把证书工具 CFSSL 和 kubectl 下发到指定节点，并下发kubeconfig配置文件 +1. 把CA 证书相关下发到指定节点的 {{ ca_dir }} 目录 +1. 最后设置基础操作系统软件和系统参数，请阅读脚本中的注释内容 + +## LB 角色-负载均衡部署 +``` bash +roles/lb +├── tasks +│   └── main.yml +└── templates + ├── haproxy.cfg.j2 + ├── haproxy.service.j2 + ├── keepalived-backup.conf.j2 + └── keepalived-master.conf.j2 +``` + +Haproxy支持四层和七层负载，稳定性好，根据官方文档，HAProxy可以跑满10Gbps-New benchmark of HAProxy at 10 Gbps using Myricom's 10GbE NICs (Myri-10G PCI-Express)；另外，openstack高可用也有用haproxy的。 + +keepalived观其名可知，保持存活，它是基于VRRP协议保证所谓的高可用或热备的，这里用来预防haproxy的单点故障。 + +keepalived与haproxy配合，实现master的高可用过程如下： + ++ 1.keepalived利用vrrp协议生成一个虚拟地址(VIP)，正常情况下VIP存活在keepalive的主节点，当主节点故障时，VIP能够漂移到keepalived的备节点，保障VIP地址可用性。 ++ 2.在keepalived的主备节点都配置相同haproxy负载配置，并且监听客户端请求在VIP的地址上，保障随时都有一个haproxy负载均衡在正常工作。并且keepalived启用对haproxy进程的存活检测，一旦主节点haproxy进程故障，VIP也能切换到备节点，从而让备节点的haproxy进行负载工作。 ++ 3.在haproxy的配置中配置多个后端真实kube-apiserver的endpoints，并启用存活监测后端kube-apiserver，如果一个kube-apiserver故障，haproxy会将其剔除负载池。 + +请在另外窗口打开[roles/lb/tasks/main.yml](../../roles/lb/tasks/main.yml) 文件，对照看以下讲解内容。 + +#### 安装haproxy + ++ 使用apt源安装 + +#### 配置haproxy [haproxy.cfg.j2](../../roles/lb/templates/haproxy.cfg.j2) +``` bash +global + log /dev/log local0 + log /dev/log local1 notice + chroot /var/lib/haproxy + stats socket /run/haproxy/admin.sock mode 660 level admin + stats timeout 30s + user haproxy + group haproxy + daemon + nbproc 1 + +defaults + log global + timeout connect 5000 + timeout client 50000 + timeout server 50000 + +listen kube-master + bind 0.0.0.0:{{ KUBE_APISERVER.split(':')[2] }} + mode tcp + option tcplog + balance source + server s1 {{ master1 }} check inter 10000 fall 2 rise 2 weight 1 + server s2 {{ master2 }} check inter 10000 fall 2 rise 2 weight 1 +``` +如果用apt安装的话，可以在/usr/share/doc/haproxy目录下找到配置指南configuration.txt.gz，全局和默认配置这里不展开，关注`listen` 代理设置模块，各项配置说明： ++ 名称 kube-master ++ bind 监听客户端请求的地址/端口，保证监听master的VIP地址和端口 ++ mode 选择四层负载模式 (当然你也可以选择七层负载，请查阅指南，适当调整) ++ balance 选择负载算法 (负载算法也有很多供选择) ++ server 配置master节点真实的endpoits，必须与 [hosts文件](../../example/hosts.m-masters.example)对应设置 + +#### 安装keepalived + ++ 使用apt源安装 + +#### 配置keepalived主节点 [keepalived-master.conf.j2](../../roles/lb/templates/keepalived-master.conf.j2) +``` bash +global_defs { + router_id lb-master +} + +vrrp_script check-haproxy { + script "killall -0 haproxy" + interval 5 + weight -30 +} + +vrrp_instance VI-kube-master { + state MASTER + priority 120 + dont_track_primary + interface {{ LB_IF }} + virtual_router_id {{ ROUTER_ID }} + advert_int 3 + track_script { + check-haproxy + } + virtual_ipaddress { + {{ MASTER_IP }} + } +} +``` ++ vrrp_script 定义了监测haproxy进程的脚本，利用shell 脚本`killall -0 haproxy` 进行检测进程是否存活，如果进程不存在，根据`weight -30`设置将主节点优先级降低30，这样原先备节点将变成主节点。 ++ vrrp_instance 定义了vrrp组，包括优先级、使用端口、router_id、心跳频率、检测脚本、虚拟地址VIP等 ++ 特别注意 `virtual_router_id` 标识了一个 VRRP组，在同网段下必须唯一，否则出现 `Keepalived_vrrp: bogus VRRP packet received on eth0 !!!`类似报错 + +#### 配置keepalived备节点 [keepalived-backup.conf.j2](../../roles/lb/templates/keepalived-backup.conf.j2) +``` bash +global_defs { + router_id lb-backup +} + +vrrp_instance VI-kube-master { + state BACKUP + priority 110 + dont_track_primary + interface {{ LB_IF }} + virtual_router_id {{ ROUTER_ID }} + advert_int 3 + virtual_ipaddress { + {{ MASTER_IP }} + } +} +``` ++ 备节点的配置类似主节点，除了优先级和检测脚本，其他如 `virtual_router_id` `advert_int` `virtual_ipaddress`必须与主节点一致 + +### 启动 keepalived 和 haproxy 后验证 + ++ lb 节点验证 + +``` bash +systemctl status haproxy # 检查进程状态 +journalctl -u haproxy # 检查进程日志是否有报错信息 +systemctl status keepalived # 检查进程状态 +journalctl -u keepalived # 检查进程日志是否有报错信息 +netstat -antlp|grep 8443 # 检查tcp端口是否监听 +``` ++ 在 keepalived 主节点 + +``` bash +ip a # 检查 master的 VIP地址是否存在 +``` +### keepalived 主备切换演练 + +1. 尝试关闭 keepalived主节点上的 haproxy进程，然后在keepalived 备节点上查看 master的 VIP地址是否能够漂移过来，并依次检查上一步中的验证项。 +1. 尝试直接关闭 keepalived 主节点系统，检查各验证项。 + +[后一篇](02-install_etcd.md) diff --git a/docs/setup/02-install_etcd.md b/docs/setup/02-install_etcd.md new file mode 100644 index 0000000..23d5ee0 --- /dev/null +++ b/docs/setup/02-install_etcd.md @@ -0,0 +1,124 @@ +## 02-安装etcd集群 + +kuberntes 系统使用 etcd 存储所有数据，是最重要的组件之一，注意 etcd集群只能有奇数个节点(1,3,5...)，本文档使用3个节点做集群。 + +请在另外窗口打开[roles/etcd/tasks/main.yml](../../roles/etcd/tasks/main.yml) 文件，对照看以下讲解内容。 + +### 下载etcd/etcdctl 二进制文件、创建证书目录 + +https://github.com/etcd-io/etcd/releases + +### 创建etcd证书请求 [etcd-csr.json.j2](../../roles/etcd/templates/etcd-csr.json.j2) + +首先判断下是否etcd 证书已经存在，如果已经存在就跳过证书生成步骤 + +``` bash +{ + "CN": "etcd", + "hosts": [ + "127.0.0.1", + "{{ inventory_hostname }}" + ], + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "CN", + "ST": "HangZhou", + "L": "XS", + "O": "k8s", + "OU": "System" + } + ] +} +``` ++ etcd使用对等证书，hosts 字段必须指定授权使用该证书的 etcd 节点 IP + +### 创建证书和私钥 + +``` bash +cd /etc/etcd/ssl && {{ bin_dir }}/cfssl gencert \ + -ca={{ ca_dir }}/ca.pem \ + -ca-key={{ ca_dir }}/ca-key.pem \ + -config={{ ca_dir }}/ca-config.json \ + -profile=kubernetes etcd-csr.json | {{ bin_dir }}/cfssljson -bare etcd +``` + +### 创建etcd 服务文件 [etcd.service.j2](../../roles/etcd/templates/etcd.service.j2) + +先创建工作目录 /var/lib/etcd/ + +``` bash +[Unit] +Description=Etcd Server +After=network.target +After=network-online.target +Wants=network-online.target +Documentation=https://github.com/coreos + +[Service] +Type=notify +WorkingDirectory=/var/lib/etcd/ +ExecStart={{ bin_dir }}/etcd \ + --name={{ NODE_NAME }} \ + --cert-file=/etc/etcd/ssl/etcd.pem \ + --key-file=/etc/etcd/ssl/etcd-key.pem \ + --peer-cert-file=/etc/etcd/ssl/etcd.pem \ + --peer-key-file=/etc/etcd/ssl/etcd-key.pem \ + --trusted-ca-file={{ ca_dir }}/ca.pem \ + --peer-trusted-ca-file={{ ca_dir }}/ca.pem \ + --initial-advertise-peer-urls=https://{{ inventory_hostname }}:2380 \ + --listen-peer-urls=https://{{ inventory_hostname }}:2380 \ + --listen-client-urls=https://{{ inventory_hostname }}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls=https://{{ inventory_hostname }}:2379 \ + --initial-cluster-token=etcd-cluster-0 \ + --initial-cluster={{ ETCD_NODES }} \ + --initial-cluster-state=new \ + --data-dir=/var/lib/etcd +Restart=on-failure +RestartSec=5 +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` ++ 完整参数列表请使用 `etcd --help` 查询 ++ 注意etcd 即需要服务器证书也需要客户端证书，这里为方便使用一个peer 证书代替两个证书，更多证书相关请阅读 [01-创建CA证书和环境配置](01-CA_and_prerequisite.md) ++ 注意{{ }} 中的参数与ansible hosts文件中设置对应 ++ `--initial-cluster-state` 值为 `new` 时，`--name` 的参数值必须位于 `--initial-cluster` 列表中； + +### 启动etcd服务 + +``` bash +systemctl daemon-reload && systemctl enable etcd && systemctl start etcd +``` + +### 验证etcd集群状态 + ++ systemctl status etcd 查看服务状态 ++ journalctl -u etcd 查看运行日志 ++ 在任一 etcd 集群节点上执行如下命令 + +``` bash +# 根据hosts中配置设置shell变量 $NODE_IPS +export NODE_IPS="192.168.1.1 192.168.1.2 192.168.1.3" +for ip in ${NODE_IPS}; do + ETCDCTL_API=3 etcdctl \ + --endpoints=https://${ip}:2379 \ + --cacert=/etc/kubernetes/ssl/ca.pem \ + --cert=/etc/etcd/ssl/etcd.pem \ + --key=/etc/etcd/ssl/etcd-key.pem \ + endpoint health; done +``` +预期结果： + +``` text +https://192.168.1.1:2379 is healthy: successfully committed proposal: took = 2.210885ms +https://192.168.1.2:2379 is healthy: successfully committed proposal: took = 2.784043ms +https://192.168.1.3:2379 is healthy: successfully committed proposal: took = 3.275709ms +``` +三台 etcd 的输出均为 healthy 时表示集群服务正常。 + +[后一篇](03-install_docker.md) diff --git a/docs/setup/03-install_docker.md b/docs/setup/03-install_docker.md new file mode 100644 index 0000000..9d1dd8c --- /dev/null +++ b/docs/setup/03-install_docker.md @@ -0,0 +1,159 @@ +## 03-安装docker服务 + +``` bash +roles/docker/ +├── files +│   ├── daemon.json +│   ├── docker +│   └── docker-tag +├── tasks +│   └── main.yml +└── templates + └── docker.service.j2 +``` + +请在另外窗口打开[roles/docker/tasks/main.yml](../../roles/docker/tasks/main.yml) 文件，对照看以下讲解内容。 + +### 创建docker的systemd unit文件 + +``` bash +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.io + +[Service] +Environment="PATH={{ bin_dir }}:/bin:/sbin:/usr/bin:/usr/sbin" +ExecStart={{ bin_dir }}/dockerd +ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT +ExecReload=/bin/kill -s HUP $MAINPID +Restart=on-failure +RestartSec=5 +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +Delegate=yes +KillMode=process + +[Install] +WantedBy=multi-user.target +``` ++ dockerd 运行时会调用其它 docker 命令，如 docker-proxy，所以需要将 docker 命令所在的目录加到 PATH 环境变量中； ++ docker 从 1.13 版本开始，将`iptables` 的`filter` 表的`FORWARD` 链的默认策略设置为`DROP`，从而导致 ping 其它 Node 上的 Pod IP 失败，因此必须在 `filter` 表的`FORWARD` 链增加一条默认允许规则 `iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT` ++ 运行`dockerd --help` 查看所有可配置参数，确保默认开启 `--iptables` 和 `--ip-masq` 选项 + +### 配置国内镜像加速 + +从国内下载docker官方仓库镜像非常缓慢，所以对于k8s集群来说配置镜像加速非常重要，配置 `/etc/docker/daemon.json` + +``` bash +{ + "registry-mirrors": ["https://registry.docker-cn.com"], + "max-concurrent-downloads": 10, + "log-driver": "json-file", + "log-level": "warn", + "log-opts": { + "max-size": "10m", + "max-file": "3" + } +} +``` + +这将在后续部署calico下载 calico/node镜像和kubedns/heapster/dashboard镜像时起到重要加速效果。 + +由于K8S的官方镜像存放在`gcr.io`仓库，因此这个镜像加速对K8S的官方镜像没有效果；好在`Docker Hub`上有很多K8S镜像的转存，而`Docker Hub`上的镜像可以加速。这里推荐两个K8S镜像的`Docker Hub`项目,几乎能找到所有K8S相关的镜像，而且更新及时，感谢维护者的辛勤付出！ + ++ [mirrorgooglecontainers](https://hub.docker.com/u/mirrorgooglecontainers/) ++ [anjia0532](https://hub.docker.com/u/anjia0532/), [项目github地址](https://github.com/anjia0532/gcr.io_mirror) + +当然对于企业内部应用的docker镜像，想要在K8S平台运行的话，特别是结合开发`CI/CD` 流程，肯定是需要部署私有镜像仓库的，后续会简单提到 `Harbor`的部署。 + +另外，daemon.json配置中也配置了docker 容器日志相关参数，设置单个容器日志超过10M则进行回卷，回卷的副本数超过3个就进行清理。 + +### 清理 iptables + +因为后续`calico`网络、`kube-proxy`等将大量使用 iptables规则，安装前清空所有`iptables`策略规则；常见发行版`Ubuntu`的 `ufw` 和 `CentOS`的 `firewalld`等基于`iptables`的防火墙最好直接卸载，避免不必要的冲突。 + +``` bash +iptables -F && iptables -X \ + && iptables -F -t nat && iptables -X -t nat \ + && iptables -F -t raw && iptables -X -t raw \ + && iptables -F -t mangle && iptables -X -t mangle +``` ++ calico 网络支持 `network-policy`，使用的`calico-kube-controllers` 会使用到`iptables` 所有的四个表 `filter` `nat` `raw` `mangle`，所以一并清理 + +### 启动 docker + +``` bash +systemctl daemon-reload && systemctl enable docker && systemctl start docker +``` + +### 可选-安装docker查询镜像 tag的小工具 + +docker官方目前没有提供在命令行直接查询某个镜像的tag信息的方式，网上找来一个脚本工具，使用很方便。 + +``` bash +$ docker-tag library/ubuntu +"14.04" +"16.04" +"17.04" +"latest" +"trusty" +"trusty-20171117" +"xenial" +"xenial-20171114" +"zesty" +"zesty-20171114" +$ docker-tag mirrorgooglecontainers/kubernetes-dashboard-amd64 +"v0.1.0" +"v1.0.0" +"v1.0.0-beta1" +"v1.0.1" +"v1.1.0-beta1" +"v1.1.0-beta2" +"v1.1.0-beta3" +"v1.7.0" +"v1.7.1" +"v1.8.0" +``` ++ 需要先apt安装轻量JSON处理程序 `jq` ++ 然后下载脚本即可使用 ++ 脚本很简单，就一行命令如下 + +``` bash +#!/bin/bash +curl -s -S "https://registry.hub.docker.com/v2/repositories/$@/tags/" | jq '."results"[]["name"]' |sort +``` ++ 对于 CentOS7 安装 `jq` 稍微费力一点，需要启用 `EPEL` 源 + +``` bash +wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +rpm -ivh epel-release-latest-7.noarch.rpm +yum install jq +``` + +### 验证 + +运行`ansible-playbook 03.docker.yml` 成功后可以验证 + +``` bash +systemctl status docker # 服务状态 +journalctl -u docker # 运行日志 +docker version +docker info +``` +`iptables-save|grep FORWARD` 查看 iptables filter表 FORWARD链，最后要有一个 `-A FORWARD -j ACCEPT` 保底允许规则 + +``` bash +iptables-save|grep FORWARD +:FORWARD ACCEPT [0:0] +:FORWARD DROP [0:0] +-A FORWARD -j DOCKER-USER +-A FORWARD -j DOCKER-ISOLATION +-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A FORWARD -o docker0 -j DOCKER +-A FORWARD -i docker0 ! -o docker0 -j ACCEPT +-A FORWARD -i docker0 -o docker0 -j ACCEPT +-A FORWARD -j ACCEPT +``` + +[后一篇](04-install_kube_master.md) diff --git a/docs/setup/04-install_kube_master.md b/docs/setup/04-install_kube_master.md new file mode 100644 index 0000000..fbc72b8 --- /dev/null +++ b/docs/setup/04-install_kube_master.md @@ -0,0 +1,242 @@ +## 04-安装kube-master节点 + +部署master节点主要包含三个组件`apiserver` `scheduler` `controller-manager`，其中： + +- apiserver提供集群管理的REST API接口，包括认证授权、数据校验以及集群状态变更等 + - 只有API Server才直接操作etcd + - 其他模块通过API Server查询或修改数据 + - 提供其他模块之间的数据交互和通信的枢纽 +- scheduler负责分配调度Pod到集群内的node节点 + - 监听kube-apiserver，查询还未分配Node的Pod + - 根据调度策略为这些Pod分配节点 +- controller-manager由一系列的控制器组成，它通过apiserver监控整个集群的状态，并确保集群处于预期的工作状态 + +master节点的高可用主要就是实现apiserver组件的高可用，在之前部署lb节点时候已经配置haproxy对它进行负载均衡。 + +``` text +roles/kube-master/ +├── tasks +│   └── main.yml +└── templates + ├── basic-auth.csv.j2 + ├── kube-apiserver.service.j2 + ├── kube-controller-manager.service.j2 + ├── kubernetes-csr.json.j2 + ├── kube-scheduler.service.j2 + └── token.csv.j2 +``` + +请在另外窗口打开[roles/kube-master/tasks/main.yml](../../roles/kube-master/tasks/main.yml) 文件，对照看以下讲解内容。 + +### 创建 kubernetes 证书签名请求 + +增加判断是否已经有kubernetes证书，如果是就使用原证书，跳过生成证书步骤 + +``` bash +{ + "CN": "kubernetes", + "hosts": [ + "127.0.0.1", + "{{ MASTER_IP }}", + "{{ inventory_hostname }}", + "{{ CLUSTER_KUBERNETES_SVC_IP }}", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster", + "kubernetes.default.svc.cluster.local" + ], + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "CN", + "ST": "HangZhou", + "L": "XS", + "O": "k8s", + "OU": "System" + } + ] +} +``` +- kubernetes 证书既是服务器证书，同时apiserver又作为客户端证书去访问etcd 集群；作为服务器证书需要设置hosts 指定使用该证书的IP 或域名列表，需要注意的是： + - 多主高可用集群需要把master VIP地址 {{ MASTER_IP }} 也添加进去 + - `kubectl get svc` 将看到集群中由api-server 创建的默认服务 `kubernetes`，因此也要把 `kubernetes` 服务名和各个服务域名也添加进去 +- 注意所有{{ }}变量与ansible hosts中设置的对应关系 + +### 创建基础用户名/密码认证配置 + +可选，为后续使用基础认证的场景做准备，如实现dashboard 用不同用户名登陆绑定不同的权限，后续更新dashboard的实践文档。 + +### 创建apiserver的服务配置文件 + +``` bash +[Unit] +Description=Kubernetes API Server +Documentation=https://github.com/GoogleCloudPlatform/kubernetes +After=network.target + +[Service] +ExecStart={{ bin_dir }}/kube-apiserver \ + --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook \ + --bind-address={{ inventory_hostname }} \ + --insecure-bind-address=127.0.0.1 \ + --authorization-mode=Node,RBAC \ + --kubelet-https=true \ + --kubelet-client-certificate={{ ca_dir }}/admin.pem \ + --kubelet-client-key={{ ca_dir }}/admin-key.pem \ + --anonymous-auth=false \ + --basic-auth-file={{ ca_dir }}/basic-auth.csv \ + --service-cluster-ip-range={{ SERVICE_CIDR }} \ + --service-node-port-range={{ NODE_PORT_RANGE }} \ + --tls-cert-file={{ ca_dir }}/kubernetes.pem \ + --tls-private-key-file={{ ca_dir }}/kubernetes-key.pem \ + --client-ca-file={{ ca_dir }}/ca.pem \ + --service-account-key-file={{ ca_dir }}/ca-key.pem \ + --etcd-cafile={{ ca_dir }}/ca.pem \ + --etcd-certfile={{ ca_dir }}/kubernetes.pem \ + --etcd-keyfile={{ ca_dir }}/kubernetes-key.pem \ + --etcd-servers={{ ETCD_ENDPOINTS }} \ + --enable-swagger-ui=true \ + --endpoint-reconciler-type=lease \ + --allow-privileged=true \ + --audit-log-maxage=30 \ + --audit-log-maxbackup=3 \ + --audit-log-maxsize=100 \ + --audit-log-path=/var/lib/audit.log \ + --event-ttl=1h \ + --requestheader-client-ca-file={{ ca_dir }}/ca.pem \ + --requestheader-allowed-names= \ + --requestheader-extra-headers-prefix=X-Remote-Extra- \ + --requestheader-group-headers=X-Remote-Group \ + --requestheader-username-headers=X-Remote-User \ + --proxy-client-cert-file={{ ca_dir }}/aggregator-proxy.pem \ + --proxy-client-key-file={{ ca_dir }}/aggregator-proxy-key.pem \ + --enable-aggregator-routing=true \ + --runtime-config=batch/v2alpha1=true \ + --v=2 +Restart=on-failure +RestartSec=5 +Type=notify +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` ++ Kubernetes 对 API 访问需要依次经过认证、授权和准入控制(admission controll)，认证解决用户是谁的问题，授权解决用户能做什么的问题，Admission Control则是资源管理方面的作用。 ++ 支持同时提供https（默认监听在6443端口）和http API（默认监听在127.0.0.1的8080端口），其中http API是非安全接口，不做任何认证授权机制，kube-scheduler、kube-controller-manager 一般和 kube-apiserver 部署在同一台机器上，它们使用非安全端口和 kube-apiserver通信; 其他集群外部就使用HTTPS访问 apiserver ++ 关于authorization-mode=Node,RBAC v1.7+支持Node授权，配合NodeRestriction准入控制来限制kubelet仅可访问node、endpoint、pod、service以及secret、configmap、PV和PVC等相关的资源；需要注意的是v1.7中Node 授权是默认开启的，v1.8中需要显式配置开启，否则 Node无法正常工作 ++ 缺省情况下 kubernetes 对象保存在 etcd /registry 路径下，可以通过 --etcd-prefix 参数进行调整 ++ 详细参数配置请参考`kube-apiserver --help`，关于认证、授权和准入控制请[阅读](https://github.com/feiskyer/kubernetes-handbook/blob/master/components/apiserver.md) ++ 增加了访问kubelet使用的证书配置，防止匿名访问kubelet的安全漏洞，详见[漏洞说明](../mixes/01.fix_kubelet_annoymous_access.md) + +### 创建controller-manager 的服务文件 + +``` bash +[Unit] +Description=Kubernetes Controller Manager +Documentation=https://github.com/GoogleCloudPlatform/kubernetes + +[Service] +ExecStart={{ bin_dir }}/kube-controller-manager \ + --address=127.0.0.1 \ + --master=http://127.0.0.1:8080 \ + --allocate-node-cidrs=true \ + --service-cluster-ip-range={{ SERVICE_CIDR }} \ + --cluster-cidr={{ CLUSTER_CIDR }} \ + --cluster-name=kubernetes \ + --cluster-signing-cert-file={{ ca_dir }}/ca.pem \ + --cluster-signing-key-file={{ ca_dir }}/ca-key.pem \ + --service-account-private-key-file={{ ca_dir }}/ca-key.pem \ + --root-ca-file={{ ca_dir }}/ca.pem \ + --horizontal-pod-autoscaler-use-rest-clients=true \ + --leader-elect=true \ + --v=2 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +``` ++ --address 值必须为 127.0.0.1，因为当前 kube-apiserver 期望 scheduler 和 controller-manager 在同一台机器 ++ --master=http://127.0.0.1:8080 使用非安全 8080 端口与 kube-apiserver 通信 ++ --cluster-cidr 指定 Cluster 中 Pod 的 CIDR 范围，该网段在各 Node 间必须路由可达(calico 实现) ++ --service-cluster-ip-range 参数指定 Cluster 中 Service 的CIDR范围，必须和 kube-apiserver 中的参数一致 ++ --cluster-signing-* 指定的证书和私钥文件用来签名为 TLS BootStrap 创建的证书和私钥 ++ --root-ca-file 用来对 kube-apiserver 证书进行校验，指定该参数后，才会在Pod 容器的 ServiceAccount 中放置该 CA 证书文件 ++ --leader-elect=true 使用多节点选主的方式选择主节点。只有主节点才会启动所有控制器，而其他从节点则仅执行选主算法 + +### 创建scheduler 的服务文件 + +``` bash +[Unit] +Description=Kubernetes Scheduler +Documentation=https://github.com/GoogleCloudPlatform/kubernetes + +[Service] +ExecStart={{ bin_dir }}/kube-scheduler \ + --address=127.0.0.1 \ + --master=http://127.0.0.1:8080 \ + --leader-elect=true \ + --v=2 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +``` + ++ --address 同样值必须为 127.0.0.1 ++ --master=http://127.0.0.1:8080 使用非安全 8080 端口与 kube-apiserver 通信 ++ --leader-elect=true 部署多台机器组成的 master 集群时选举产生一个处于工作状态的 kube-controller-manager 进程 + +### 在master 节点安装 node 服务: kubelet kube-proxy + +项目master 分支使用 DaemonSet 方式安装网络插件，如果master 节点不安装 kubelet 服务是无法安装网络插件的，如果 master 节点不安装网络插件，那么通过`apiserver` 方式无法访问 `dashboard` `kibana`等管理界面，[ISSUES #130](https://github.com/gjmzj/kubeasz/issues/130) + +项目v1.8 分支使用二进制方式安装网络插件，所以没有这个问题 + +``` bash +# vi 04.kube-master.yml +- hosts: kube-master + roles: + - kube-master + - kube-node + # 禁止业务 pod调度到 master节点 + tasks: + - name: 禁止业务 pod调度到 master节点 + shell: "{{ bin_dir }}/kubectl cordon {{ inventory_hostname }} " + when: DEPLOY_MODE != "allinone" + ignore_errors: true +``` +在master 节点也同时成为 node 节点后，默认业务 POD也会调度到 master节点，多主模式下这显然增加了 master节点的负载，因此可以使用 `kubectl cordon`命令禁止业务 POD调度到 master节点 + + +### master 集群的验证 + +运行 `ansible-playbook 04.kube-master.yml` 成功后，验证 master节点的主要组件： + +``` bash +# 查看进程状态 +systemctl status kube-apiserver +systemctl status kube-controller-manager +systemctl status kube-scheduler +# 查看进程运行日志 +journalctl -u kube-apiserver +journalctl -u kube-controller-manager +journalctl -u kube-scheduler +``` +执行 `kubectl get componentstatus` 可以看到 + +``` bash +NAME STATUS MESSAGE ERROR +scheduler Healthy ok +controller-manager Healthy ok +etcd-0 Healthy {"health": "true"} +etcd-2 Healthy {"health": "true"} +etcd-1 Healthy {"health": "true"} +``` + +[后一篇](05-install_kube_node.md) diff --git a/docs/setup/05-install_kube_node.md b/docs/setup/05-install_kube_node.md new file mode 100644 index 0000000..db97e60 --- /dev/null +++ b/docs/setup/05-install_kube_node.md @@ -0,0 +1,142 @@ +## 05-安装kube-node节点 + +`kube-node` 是集群中承载应用的节点，前置条件需要先部署好`kube-master`节点(因为需要操作`用户角色绑定`、`批准kubelet TLS 证书请求`等)，它需要部署如下组件： + ++ docker：运行容器 ++ calico： 配置容器网络 (或者 flannel) ++ kubelet： kube-node上最主要的组件 ++ kube-proxy： 发布应用服务与负载均衡 + +``` bash +roles/kube-node +├── tasks +│   └── main.yml +└── templates + ├── cni-default.conf.j2 + ├── kubelet.service.j2 + ├── kubelet-csr.json.j2 + └── kube-proxy.service.j2 +``` + +请在另外窗口打开[roles/kube-node/tasks/main.yml](../../roles/kube-node/tasks/main.yml) 文件，对照看以下讲解内容。 + +### 创建cni 基础网络插件配置文件 + +因为后续需要用 `DaemonSet Pod`方式运行k8s网络插件，所以kubelet.server服务必须开启cni相关参数，并且提供cni网络配置文件 + +### 创建 kubelet 的服务文件 + ++ 必须先创建工作目录 `/var/lib/kubelet` + +``` bash +[Unit] +Description=Kubernetes Kubelet +Documentation=https://github.com/GoogleCloudPlatform/kubernetes +After=docker.service +Requires=docker.service + +[Service] +WorkingDirectory=/var/lib/kubelet +#--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest +ExecStart={{ bin_dir }}/kubelet \ + --address={{ inventory_hostname }} \ + --allow-privileged=true \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --pod-manifest-path=/etc/kubernetes/manifest \ + --client-ca-file={{ ca_dir }}/ca.pem \ + --cluster-dns={{ CLUSTER_DNS_SVC_IP }} \ + --cluster-domain={{ CLUSTER_DNS_DOMAIN }} \ + --cni-bin-dir={{ bin_dir }} \ + --cni-conf-dir=/etc/cni/net.d \ + --fail-swap-on=false \ + --hairpin-mode hairpin-veth \ + --hostname-override={{ inventory_hostname }} \ + --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \ + --max-pods={{ MAX_PODS }} \ + --network-plugin=cni \ + --pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.1 \ + --register-node=true \ + --root-dir={{ KUBELET_ROOT_DIR }} \ + --tls-cert-file={{ ca_dir }}/kubelet.pem \ + --tls-private-key-file={{ ca_dir }}/kubelet-key.pem \ + --v=2 +#kubelet cAdvisor 默认在所有接口监听 4194 端口的请求, 以下iptables限制内网访问 +ExecStartPost=/sbin/iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 4194 -j ACCEPT +ExecStartPost=/sbin/iptables -A INPUT -s 172.16.0.0/12 -p tcp --dport 4194 -j ACCEPT +ExecStartPost=/sbin/iptables -A INPUT -s 192.168.0.0/16 -p tcp --dport 4194 -j ACCEPT +ExecStartPost=/sbin/iptables -A INPUT -p tcp --dport 4194 -j DROP +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +``` ++ --pod-infra-container-image 指定`基础容器`（负责创建Pod 内部共享的网络、文件系统等）镜像，**K8S每一个运行的 POD里面必然包含这个基础容器**，如果它没有运行起来那么你的POD 肯定创建不了，kubelet日志里面会看到类似 ` FailedCreatePodSandBox` 错误，可用`docker images` 查看节点是否已经下载到该镜像 ++ --cluster-dns 指定 kubedns 的 Service IP(可以先分配，后续创建 kubedns 服务时指定该 IP)，--cluster-domain 指定域名后缀，这两个参数同时指定后才会生效； ++ --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir={{ bin_dir }} 为使用cni 网络，并调用calico管理网络所需的配置 ++ --fail-swap-on=false K8S 1.8+需显示禁用这个，否则服务不能启动 ++ --client-ca-file={{ ca_dir }}/ca.pem 和 --anonymous-auth=false 关闭kubelet的匿名访问，详见[匿名访问漏洞说明](mixes/01.fix_kubelet_annoymous_access.md) + +### 创建 kube-proxy kubeconfig 文件 + +该步骤已经在 deploy节点完成，[roles/deploy/tasks/main.yml](../../roles/deploy/tasks/main.yml) + ++ 生成的kube-proxy.kubeconfig 配置文件需要移动到/etc/kubernetes/目录，后续kube-proxy服务启动参数里面需要指定 + +### 创建 kube-proxy服务文件 + +``` bash +[Unit] +Description=Kubernetes Kube-Proxy Server +Documentation=https://github.com/GoogleCloudPlatform/kubernetes +After=network.target + +[Service] +WorkingDirectory=/var/lib/kube-proxy +ExecStart={{ bin_dir }}/kube-proxy \ + --bind-address={{ inventory_hostname }} \ + --hostname-override={{ inventory_hostname }} \ + --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \ + --logtostderr=true \ + --v=2 +Restart=on-failure +RestartSec=5 +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` + ++ --hostname-override 参数值必须与 kubelet 的值一致，否则 kube-proxy 启动后会找不到该 Node，从而不会创建任何 iptables 规则 ++ 特别注意：kube-proxy 根据 --cluster-cidr 判断集群内部和外部流量，指定 --cluster-cidr 或 --masquerade-all 选项后 kube-proxy 才会对访问 Service IP 的请求做 SNAT；但是这个特性与calico 实现 network policy冲突，所以如果要用 network policy，这两个选项都不要指定。 + +### 批准kubelet 的 TLS 证书请求 + +``` bash +sleep 15 && {{ bin_dir }}/kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| xargs {{ bin_dir }}/kubectl certificate approve +``` ++ 增加15秒延时等待kubelet启动 ++ `kubectl get csr |grep 'Pending'` 找出待批准的 TLS请求 ++ `kubectl certificate approve` 批准请求 + +### 验证 node 状态 + +``` bash +systemctl status kubelet # 查看状态 +systemctl status kube-proxy +journalctl -u kubelet # 查看日志 +journalctl -u kube-proxy +``` +运行 `kubectl get node` 可以看到类似 + +``` bash +NAME STATUS ROLES AGE VERSION +192.168.1.42 Ready 2d v1.9.0 +192.168.1.43 Ready 2d v1.9.0 +192.168.1.44 Ready 2d v1.9.0 +``` + + +[后一篇](06-install_network_plugin.md) diff --git a/docs/setup/06-install_network_plugin.md b/docs/setup/06-install_network_plugin.md new file mode 100644 index 0000000..212b1a4 --- /dev/null +++ b/docs/setup/06-install_network_plugin.md @@ -0,0 +1,42 @@ +## 06-安装网络组件 + +首先回顾下K8S网络设计原则，在配置集群网络插件或者实践K8S 应用/服务部署请时刻想到这些原则： + +- 1.每个Pod都拥有一个独立IP地址，Pod内所有容器共享一个网络命名空间 +- 2.集群内所有Pod都在一个直接连通的扁平网络中，可通过IP直接访问 + - 所有容器之间无需NAT就可以直接互相访问 + - 所有Node和所有容器之间无需NAT就可以直接互相访问 + - 容器自己看到的IP跟其他容器看到的一样 +- 3.Service cluster IP尽可在集群内部访问，外部请求需要通过NodePort、LoadBalance或者Ingress来访问 + +`Container Network Interface (CNI)`是目前CNCF主推的网络模型，它由两部分组成： + +- CNI Plugin负责给容器配置网络，它包括两个基本的接口 + - 配置网络: AddNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) + - 清理网络: DelNetwork(net *NetworkConfig, rt *RuntimeConf) error +- IPAM Plugin负责给容器分配IP地址 + +Kubernetes Pod的网络是这样创建的： +- 0.每个Pod除了创建时指定的容器外，都有一个kubelet启动时指定的`基础容器`，比如：`mirrorgooglecontainers/pause-amd64` `registry.access.redhat.com/rhel7/pod-infrastructure` +- 1.首先 kubelet创建`基础容器`生成network namespace +- 2.然后 kubelet调用网络CNI driver，由它根据配置调用具体的CNI 插件 +- 3.然后 CNI 插件给`基础容器`配置网络 +- 4.最后 Pod 中其他的容器共享使用`基础容器`的网络 + +本项目基于CNI driver 调用各种网络插件来配置kubernetes的网络，常用CNI插件有 `flannel` `calico` `weave`等等，这些插件各有优势，也在互相借鉴学习优点，比如：在所有node节点都在一个二层网络时候，flannel提供hostgw实现，避免vxlan实现的udp封装开销，估计是目前最高效的；calico也针对L3 Fabric，推出了IPinIP的选项，利用了GRE隧道封装；因此这些插件都能适合很多实际应用场景。 + +项目当前内置支持的网络插件有：`calico` `cilium` `flannel` `kube-router` + +### 安装讲解 + +- [安装calico](network-plugin/calico.md) +- [安装cilium](network-plugin/cilium.md) +- [安装flannel](network-plugin/flannel.md) +- [安装kube-router](network-plugin/kube-router.md) + +### 参考 +- [kubernetes.io networking docs](https://kubernetes.io/docs/concepts/cluster-administration/networking/) +- [feiskyer-kubernetes指南网络章节](https://github.com/feiskyer/kubernetes-handbook/blob/master/zh/network/network.md) + + +[后一篇](07-install_cluster_addon.md) diff --git a/docs/setup/07-install_cluster_addon.md b/docs/setup/07-install_cluster_addon.md new file mode 100644 index 0000000..a664d43 --- /dev/null +++ b/docs/setup/07-install_cluster_addon.md @@ -0,0 +1,16 @@ +# 07-安装集群主要插件 + +目前挑选一些常用、必要的插件自动集成到安装脚本之中: +- [自动脚本](../../roles/cluster-addon/tasks/main.yml) +- 配置开关 + - 参照[配置指南](config_guide.md)，生成后在`roles/cluster-addon/defaults/main.yml`配置 + +## 脚本介绍 + +- 1.根据hosts文件中配置的`CLUSTER_DNS_SVC_IP` `CLUSTER_DNS_DOMAIN`等参数生成kubedns.yaml和coredns.yaml文件 +- 2.注册变量pod_info，pod_info用来判断现有集群是否已经运行各种插件 +- 3.根据pod_info和`配置开关`逐个进行/跳过插件安装 + + + +[后一篇](08-cluster-storage.md) diff --git a/docs/setup/08-cluster-storage.md b/docs/setup/08-cluster-storage.md new file mode 100644 index 0000000..168b654 --- /dev/null +++ b/docs/setup/08-cluster-storage.md @@ -0,0 +1,99 @@ +# K8S 集群存储 + +## 前言 +在kubernetes(k8s)中对于存储的资源抽象了两个概念，分别是PersistentVolume(PV)、PersistentVolumeClaim(PVC)。 +- PV是集群中的资源 +- PVC是对这些资源的请求。 + +如上面所说PV和PVC都只是抽象的概念，在k8s中是通过插件的方式提供具体的存储实现。目前包含有NFS、iSCSI和云提供商指定的存储系统，更多的存储实现[参考官方文档](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)。 + +这里PV又有两种提供方式: 静态或者动态。 +本篇以介绍 **NFS存储** 为例，讲解k8s 众多存储方案中的一个实现。 + +## 静态 PV +首先我们需要一个NFS服务器，用于提供底层存储。通过文档[nfs-server](../guide/nfs-server.md)，我们可以创建一个NFS服务器。 + +- 创建静态 pv，指定容量，访问模式，回收策略，存储类等；参考[这里](https://github.com/feiskyer/kubernetes-handbook/blob/master/zh/concepts/persistent-volume.md) + +``` bash +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-es-0 +spec: + capacity: + storage: 4Gi + accessModes: + - ReadWriteMany + volumeMode: Filesystem + persistentVolumeReclaimPolicy: Recycle + storageClassName: "es-storage-class" + nfs: + # 根据实际共享目录修改 + path: /share/es0 + # 根据实际 nfs服务器地址修改 + server: 192.168.1.208 +``` +- 创建 pvc即可绑定使用上述 pv了，具体请看后文 test pod例子 + +## 创建动态PV + +在一个工作k8s 集群中，`PVC`请求会很多，如果每次都需要管理员手动去创建对应的 `PV`资源，那就很不方便；因此 K8S还提供了多种 `provisioner`来动态创建 `PV`，不仅节省了管理员的时间，还可以根据`StorageClasses`封装不同类型的存储供 PVC 选用。 + +项目中的 `role: cluster-storage`目前支持自建nfs 和aliyun_nas 的动态`provisioner` + +- 1.编辑自定义配置文件：上述命令执行后生成的roles/cluster-storage/defaults/main.yml + +``` bash +# 比如创建nfs provisioner +storage: + nfs: + enabled: "yes" + server: "192.168.1.8" + server_path: "/data/nfs" + storage_class: "class-nfs-01" + provisioner_name: "nfs-provisioner-01" +``` +- 3.创建 nfs provisioner + +``` bash +$ ansible-playbook /etc/ansible/roles/cluster-storage/cluster-storage.yml +# 执行成功后验证 +$ kubectl get pod --all-namespaces |grep nfs-prov +kube-system nfs-provisioner-01-6b7fbbf9d4-bh8lh 1/1 Running 0 1d +``` +**注意** k8s集群可以使用多个nfs provisioner，重复上述步骤2 修改使用不同的`nfs server` `nfs_storage_class` `nfs_provisioner_name`后执行步骤3创建即可。 + +## 验证使用动态 PV + +切换到项目`manifests/storage`目录，编辑`test.yaml`文件，根据前文配置情况修改`storageClassName`即可；然后执行以下命令进行创建： + +``` bash +$ kubectl apply -f test.yaml + +# 验证测试pod +$ kubectl get pod --all-namespaces |grep test +default test 1/1 Running 0 1m + +# 验证自动创建的pv 资源， +$ kubectl get pv +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-8f1b4ced-92d2-11e8-a41f-5254008ec7c0 1Mi RWX Delete Bound default/test-claim nfs-dynamic-class-01 3m + +# 验证PVC已经绑定成功：STATUS字段为 Bound +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +test-claim Bound pvc-8f1b4ced-92d2-11e8-a41f-5254008ec7c0 1Mi RWX nfs-dynamic-class-01 3m +``` + +另外，Pod启动完成后，在挂载的目录中创建一个`SUCCESS`文件。我们可以到NFS服务器去看下： + +``` +. +└── default-test-claim-pvc-a877172b-5f49-11e8-b675-d8cb8ae6325a + └── SUCCESS +``` +如上，可以发现挂载的时候，nfs-client根据PVC自动创建了一个目录，我们Pod中挂载的`/mnt`，实际引用的就是该目录，而我们在`/mnt`下创建的`SUCCESS`文件，也自动写入到了这里。 + +# 后续 +后面当我们需要为上层应用提供持久化存储时，只需要提供`StorageClass`即可。很多应用都会根据`StorageClass`来创建他们的所需的PVC, 最后再把PVC挂载到他们的Deployment或StatefulSet中使用，比如：efk、jenkins等 diff --git a/docs/setup/config_guide.md b/docs/setup/config_guide.md new file mode 100644 index 0000000..1b4168a --- /dev/null +++ b/docs/setup/config_guide.md @@ -0,0 +1,31 @@ +# 个性化集群参数配置 + +对于刚接触项目者，如"快速指南"说明，只需要： + +- **1** 个配置：`/etc/ansible/hosts` +- **1** 键安装：`ansible-playbook /etc/ansilbe/90.setup.yml` + +具体来讲 `kubeasz`创建集群主要在以下两个地方进行配置： + +- ansible hosts 文件（模板在examples目录）：集群主要节点定义和主要参数配置、全局变量 +- roles/xxx/defaults/main.yml 文件：其他参数配置或者部分组件附加参数 + +## ansible hosts + +项目在[快速指南](quickStart.md)或者[集群规划与安装概览](00-planning_and_overall_intro.md)已经介绍过，主要包括集群节点定义和集群范围的主要参数配置；目前提供四种集群部署模板。 + +- 尽量保持配置简单灵活 +- 尽量保持配置项稳定 + +## roles/xxx/defaults/main.yml + +主要包括集群某个具体组件的个性化配置，具体组件的配置项可能会不断增加； + +- 可以在不做任何配置更改情况下使用默认值创建集群 +- 可以根据实际需要配置 k8s 集群，常用举例 + - 配置 kube-proxy 使用 ipvs：修改 roles/kube-node/defaults/main.yml 变量 PROXY_MODE: "ipvs" + - 配置 lb 节点负载均衡算法：修改 roles/lb/defaults/main.yml 变量 BALANCE_ALG: "roundrobin" + - 配置 docker 国内镜像加速站点：修改 roles/docker/defaults/main.yml 相关变量 + - 配置 apiserver 支持公网域名：修改 roles/kube-master/defaults/main.yml 相关变量 + - 配置 flannel 使用镜像版本：修改 roles/flannel/defaults/main.yml 相关变量 + - 配置选择不同 addon 组件：修改roles/cluster-addon/defaults/main.yml diff --git a/docs/setup/docker_kubeasz.md b/docs/setup/docker_kubeasz.md new file mode 100644 index 0000000..0bdb4cb --- /dev/null +++ b/docs/setup/docker_kubeasz.md @@ -0,0 +1,73 @@ +# 容器化运行 kubeasz + +## TL;DR; + +- 1.本机安装 docker (略) +- 2.配置 ssh 密钥登陆集群节点 + +``` bash +ssh-keygen -t rsa -b 2048 回车 回车 回车 +ssh-copy-id $IP # $IP 为所有节点地址包括自身，按照提示输入 yes 和 root 密码 +``` + +- 3.下载 kubeasz docker 镜像并运行 + +``` bash +export KUBEASZ_VER=1.0.0rc1 +docker pull jmgao1983/kubeasz:$KUBEASZ_VER +wget https://github.com/gjmzj/kubeasz/releases/download/$KUBEASZ_VER/kubeasz-docker +bash kubeasz-docker start $KUBEASZ_VER +``` + +- 4.在 kubeasz 容器中创建 k8s 集群，步骤与非容器方式创建类似 + +``` bash +# 进入容器后，在 /etc/ansible 目录配置 hosts等，然后创建集群（与非容器方式一致） +docker exec -it kubeasz sh +# 举例1：快速创建单节点集群 +docker exec -t kubeasz easzctl start aio +``` + +## 验证 + +使用容器化安装成功后，可以在 **容器内** 或者 **宿主机** 上执行 kubectl 命令验证集群状态。 + +## kubeasz 镜像介绍 + +镜像描述文件 dockerfiles/kubeasz/Dockerfile，它基于 ansible 镜像（dockerfiles/ansible/Dockerfile），主要包含 kubeasz 项目代码和 k8s 集群安装所需二进制文件。 + +- 在本地创建 kubeasz 镜像，由于镜像较大，可以按以下步骤在本地创建 + +``` bash +cd /etc/ansible/dockerfiles/kubeasz +# 克隆代码 +git clone https://github.com/gjmzj/kubeasz.git +# 手动下载二进制文件放入上述 git clone 完成目录 kubeasz/bin +docker build -t kubeasz:$TAG . +``` + +## 容器运行讲解 + +容器启动脚本详见文件 tools/kubeasz_docker + +``` bash +docker run --detach \ + --name kubeasz \ + --restart always \ + --env KUBEASZ_DOCKER_HOST=$HOST_IP \ + --volume /etc/ansible:/etc/ansible \ + --volume /root/.kube:/root/.kube \ + --volume /root/.ssh/id_rsa:/root/.ssh/id_rsa:ro \ + --volume /root/.ssh/id_rsa.pub:/root/.ssh/id_rsa.pub:ro \ + --volume /root/.ssh/known_hosts:/root/.ssh/known_hosts:ro \ + $KUBEASZ_DOCKER_VER +``` + +- --env KUBEASZ_DOCKER_HOST=$HOST_IP 传递这个参数是为了快速在本机安装aio集群 +- --volume /etc/ansible:/etc/ansible 挂载本地目录，这样可以在宿主机上修改集群配置，然后在容器内执行 ansible 安装 +- --volume /root/.kube:/root/.kube 容器内与主机共享 kubeconfig，这样都可以执行 kubectl 命令 +- --volume /root/.ssh/id_rsa:/root/.ssh/id_rsa:ro 等三个 volume 挂载保证：如果宿主机配置了免密码登陆所有集群节点，那么容器内也可以免密码登陆所有节点 + +## 参考 + +- ansible 容器镜像制作: https://github.com/William-Yeh/docker-ansible diff --git a/docs/setup/kubeasz_on_public_cloud.md b/docs/setup/kubeasz_on_public_cloud.md new file mode 100644 index 0000000..a4c8f3d --- /dev/null +++ b/docs/setup/kubeasz_on_public_cloud.md @@ -0,0 +1,79 @@ +# 公有云上部署 kubeasz + +在公有云上使用`kubeasz`部署`k8s`集群需要注意以下几点： + +1. 注意虚机的安全组规则配置，一般集群内部节点之间端口放开即可; + +2. 部分`k8s`网络组件受限，一般可以选择 flannel (vxlan模式)、calico（开启ipinip）; + +3. 无法自由创建`lb`节点，一般使用云负载均衡（内网）四层TCP负载模式; + +4. 部分云厂商负载均衡使用四层负载模式时不支持添加进后端云服务器池的 ECS 既作为 Real Server，又作为客户端向所在的 SLB 实例发送请求；因此注意不要在 master节点执行 kubectl，会出现时通时不通的情况； + +## 在公有云上部署多主多节点集群 + +- 单节点和单主多节点集群的节点规划与自有环境没有差异 + +- 多主多节点集群节点规划不需要lb节点 + +其他在公有云上的安装步骤与自有环境没有差异，节点规划可以参考 [example/hosts.cloud.example](../../example/hosts.cloud.example)，如下：（避免deploy节点同时作为master节点） + +``` bash +# 集群部署节点：一般为运行ansible 脚本的节点 +# 变量 NTP_ENABLED (=yes/no) 设置集群是否安装 chrony 时间同步, 公有云上虚机不需要 +[deploy] +10.1.0.160 NTP_ENABLED=no + +# etcd集群请提供如下NODE_NAME，注意etcd集群必须是1,3,5,7...奇数个节点 +[etcd] +10.1.0.160 NODE_NAME=etcd1 +10.1.0.161 NODE_NAME=etcd2 +10.1.0.162 NODE_NAME=etcd3 + +[kube-master] +10.1.0.161 +10.1.0.162 + +# 公有云上一般都有提供负载均衡产品，且不允许自己创建，lb 节点留空，仅保留组名 +[lb] + +[kube-node] +10.1.0.160 +10.1.0.163 + +# 参数 NEW_INSTALL：yes表示新建，no表示使用已有harbor服务器 +[harbor] +#10.1.0.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no + +... +``` ++ 创建云负载均衡，例如阿里云slb如下： + +``` bash +1. 首先创建SLB，注意选择【可用区】，【实例类型】可以先选‘私网’，【网络类型】专有网络，【虚拟交换机】跟你k8s集群节点同一交换机 +2. 配置【协议&监听】TCP 【端口】8443，【后端服务器】即 master 节点服务器，端口 6443 +3. 配置完成，记下负载均衡的内部地址（例如 10.1.0.200） +``` ++ 继续配置 ansible hosts，设置`MASTER_IP` 为刚才创建的SLB地址 + +``` bash +[all:vars] +# ---------集群主要参数--------------- +#集群部署模式：allinone, single-master, multi-master +DEPLOY_MODE=multi-master + +# 创建内网云负载均衡，然后配置：前端监听 tcp 8443，后端 tcp 6443，后端节点即 master 节点 +MASTER_IP="10.1.0.200" # 即负载均衡内网地址 +KUBE_APISERVER="https://{{ MASTER_IP }}:8443" + +# 集群网络插件，目前支持calico, flannel +CLUSTER_NETWORK="flannel" + +... +``` ++ 一步创建集群 `ansible-playbook /etc/ansible/90.setup.yml` + +### 其他资料 + +另外由[li-sen](https://github.com/li-sen)分享的[kubeasz-阿里云vpc部署记录](https://li-sen.github.io/post/blog-wiki/2018-09-27-k8s-kubeasz-%E9%98%BF%E9%87%8C%E4%BA%91vpc%E9%83%A8%E7%BD%B2%E8%AE%B0%E5%BD%95/)：介绍了阿里云上自建高可用k8s集群碰过的问题与解决，主要是使用一台haproxy中转解决slb的限制问题。 + diff --git a/docs/setup/network-plugin/calico-bgp-rr.md b/docs/setup/network-plugin/calico-bgp-rr.md new file mode 100644 index 0000000..d6a4099 --- /dev/null +++ b/docs/setup/network-plugin/calico-bgp-rr.md @@ -0,0 +1,263 @@ +# calico 配置 BGP Route Reflectors + +`Calico`作为`k8s`的一个流行网络插件，它依赖`BGP`路由协议实现集群节点上的`POD`路由互通；而路由互通的前提是节点间建立 BGP Peer 连接。BGP 路由反射器（Route Reflectors，简称 RR）可以简化集群BGP Peer的连接方式，它是解决BGP扩展性问题的有效方式；具体来说： + +- 没有 RR 时，所有节点之间需要两两建立连接（IBGP全互联），节点数量增加将导致连接数剧增、资源占用剧增 +- 引入 RR 后，其他 BGP 路由器只需要与它建立连接并交换路由信息，节点数量增加连接数只是线性增加，节省系统资源 + +calico-node 版本 v3.3 开始支持内建路由反射器，非常方便，因此使用 calico 作为网络插件可以支持大规模节点数的`K8S`集群。 + +本文档主要讲解配置 BGP Route Reflectors，建议首先阅读[基础calico文档](calico.md)。 + +## 前提条件 + +实验环境为按照kubeasz安装的2主2从集群，calico 版本 v3.3.2 + +``` +$ kubectl get node +NAME STATUS ROLES AGE VERSION +192.168.1.1 Ready,SchedulingDisabled master 178m v1.13.1 +192.168.1.2 Ready,SchedulingDisabled master 178m v1.13.1 +192.168.1.3 Ready node 178m v1.13.1 +192.168.1.4 Ready node 178m v1.13.1 +$ kubectl get pod -n kube-system -o wide | grep calico +calico-kube-controllers-77487546bd-jqrlc 1/1 Running 0 179m 192.168.1.3 192.168.1.3 +calico-node-67t5m 2/2 Running 0 179m 192.168.1.1 192.168.1.1 +calico-node-drmhq 2/2 Running 0 179m 192.168.1.2 192.168.1.2 +calico-node-rjtkv 2/2 Running 0 179m 192.168.1.4 192.168.1.4 +calico-node-xtspl 2/2 Running 0 179m 192.168.1.3 192.168.1.3 +``` +查看当前集群中BGP连接情况：可以看到集群中4个节点两两建立了 BGP 连接 + +``` +$ ansible all -m shell -a '/opt/kube/bin/calicoctl node status' +192.168.1.3 | SUCCESS | rc=0 >> +Calico process is running. + +IPv4 BGP status ++--------------+-------------------+-------+----------+-------------+ +| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | ++--------------+-------------------+-------+----------+-------------+ +| 192.168.1.1 | node-to-node mesh | up | 03:08:20 | Established | +| 192.168.1.2 | node-to-node mesh | up | 03:08:18 | Established | +| 192.168.1.4 | node-to-node mesh | up | 03:08:19 | Established | ++--------------+-------------------+-------+----------+-------------+ + +IPv6 BGP status +No IPv6 peers found. + +192.168.1.2 | SUCCESS | rc=0 >> +Calico process is running. + +IPv4 BGP status ++--------------+-------------------+-------+----------+-------------+ +| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | ++--------------+-------------------+-------+----------+-------------+ +| 192.168.1.4 | node-to-node mesh | up | 03:08:17 | Established | +| 192.168.1.3 | node-to-node mesh | up | 03:08:18 | Established | +| 192.168.1.1 | node-to-node mesh | up | 03:08:20 | Established | ++--------------+-------------------+-------+----------+-------------+ + +IPv6 BGP status +No IPv6 peers found. + +192.168.1.1 | SUCCESS | rc=0 >> +Calico process is running. + +IPv4 BGP status ++--------------+-------------------+-------+----------+-------------+ +| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | ++--------------+-------------------+-------+----------+-------------+ +| 192.168.1.2 | node-to-node mesh | up | 03:08:21 | Established | +| 192.168.1.3 | node-to-node mesh | up | 03:08:21 | Established | +| 192.168.1.4 | node-to-node mesh | up | 03:08:21 | Established | ++--------------+-------------------+-------+----------+-------------+ + +IPv6 BGP status +No IPv6 peers found. + +192.168.1.4 | SUCCESS | rc=0 >> +Calico process is running. + +IPv4 BGP status ++--------------+-------------------+-------+----------+-------------+ +| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | ++--------------+-------------------+-------+----------+-------------+ +| 192.168.1.2 | node-to-node mesh | up | 03:08:17 | Established | +| 192.168.1.3 | node-to-node mesh | up | 03:08:19 | Established | +| 192.168.1.1 | node-to-node mesh | up | 03:08:20 | Established | ++--------------+-------------------+-------+----------+-------------+ + +IPv6 BGP status +No IPv6 peers found. +``` +## 配置全局禁用全连接（BGP full mesh） + +``` +$ cat << EOF | calicoctl create -f - +apiVersion: projectcalico.org/v3 +kind: BGPConfiguration +metadata: + name: default +spec: + logSeverityScreen: Info + nodeToNodeMeshEnabled: false + asNumber: 64512 +EOF +``` + +上述命令配置完成后，再次使用命令`ansible all -m shell -a '/opt/kube/bin/calicoctl node status'`查看，可以看到之前所有的bgp连接都消失了。 + +## 配置 BGP node 与 Route Reflector 的连接建立规则 + +``` bash +$ cat << EOF | calicoctl create -f - +kind: BGPPeer +apiVersion: projectcalico.org/v3 +metadata: + name: peer-to-rrs +spec: + # 规则1：普通 bgp node 与 rr 建立连接 + nodeSelector: !has(i-am-a-route-reflector) + peerSelector: has(i-am-a-route-reflector) + +--- +kind: BGPPeer +apiVersion: projectcalico.org/v3 +metadata: + name: rr-mesh +spec: + # 规则2：route reflectors 之间也建立连接 + nodeSelector: has(i-am-a-route-reflector) + peerSelector: has(i-am-a-route-reflector) +EOF +``` + +上述命令配置完成后，使用命令：`calicoctl get bgppeer` `calicoctl get bgppeer rr-mesh -o yaml` 检查配置是否正确。 + +## 选择并配置 Route Reflector 节点 + +首先查看当前集群中的节点： + +``` +$ calicoctl get node -o wide +NAME ASN IPV4 IPV6 +k8s401 (64512) 192.168.1.1/24 +k8s402 (64512) 192.168.1.2/24 +k8s403 (64512) 192.168.1.3/24 +k8s404 (64512) 192.168.1.4/24 +``` + +可以在集群中选择1个或多个节点作为 rr 节点，这里先选择节点：k8s401 + +``` bash +# 1.先导出 node k8s401 的配置，准备修改 +$ calicoctl get node k8s401 --export -o yaml |tee rr01.yml +apiVersion: projectcalico.org/v3 +kind: Node +metadata: + creationTimestamp: null + name: k8s401 +spec: + bgp: + ipv4Address: 192.168.1.1/24 + ipv4IPIPTunnelAddr: 172.20.7.128 + orchRefs: + - nodeName: 192.168.1.1 + orchestrator: k8s + +# 2.修改上述 rr01.yml 的配置如下 +apiVersion: projectcalico.org/v3 +kind: Node +metadata: + creationTimestamp: null + name: k8s401 + labels: + # 设置标签 + i-am-a-route-reflector: true +spec: + bgp: + ipv4Address: 192.168.1.1/24 + ipv4IPIPTunnelAddr: 172.20.7.128 + # 设置集群ID + routeReflectorClusterID: 224.0.0.1 + orchRefs: + - nodeName: 192.168.1.1 + orchestrator: k8s + +# 3.应用修改后的 rr node 配置 +$ calicoctl apply -f rr01.yml +``` + +## 查看增加 rr 之后的bgp 连接情况 + +``` +$ ansible all -m shell -a '/opt/kube/bin/calicoctl node status' +192.168.1.4 | SUCCESS | rc=0 >> +Calico process is running. + +IPv4 BGP status ++--------------+-----------+-------+----------+-------------+ +| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | ++--------------+-----------+-------+----------+-------------+ +| 192.168.1.1 | global | up | 11:02:55 | Established | ++--------------+-----------+-------+----------+-------------+ + +IPv6 BGP status +No IPv6 peers found. + +192.168.1.3 | SUCCESS | rc=0 >> +Calico process is running. + +IPv4 BGP status ++--------------+-----------+-------+----------+-------------+ +| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | ++--------------+-----------+-------+----------+-------------+ +| 192.168.1.1 | global | up | 11:02:55 | Established | ++--------------+-----------+-------+----------+-------------+ + +IPv6 BGP status +No IPv6 peers found. + +192.168.1.1 | SUCCESS | rc=0 >> +Calico process is running. + +IPv4 BGP status ++--------------+---------------+-------+----------+-------------+ +| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | ++--------------+---------------+-------+----------+-------------+ +| 192.168.1.2 | node specific | up | 11:02:55 | Established | +| 192.168.1.3 | node specific | up | 11:02:55 | Established | +| 192.168.1.4 | node specific | up | 11:02:55 | Established | ++--------------+---------------+-------+----------+-------------+ + +IPv6 BGP status +No IPv6 peers found. + +192.168.1.2 | SUCCESS | rc=0 >> +Calico process is running. + +IPv4 BGP status ++--------------+-----------+-------+----------+-------------+ +| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | ++--------------+-----------+-------+----------+-------------+ +| 192.168.1.1 | global | up | 11:02:55 | Established | ++--------------+-----------+-------+----------+-------------+ + +IPv6 BGP status +No IPv6 peers found. +``` +可以看到所有其他节点都与所选rr节点建立bgp连接。 + +## 再增加一个 rr 节点 + +步骤同上述选择第1个 rr 节点，这里省略；添加成功后可以看到所有其他节点都与两个rr节点建立bgp连接，两个rr节点之间也建立bgp连接。 + +- 对于节点数较多的`K8S`集群建议配置3-4个 RR 节点 + +## 参考文档 + +- 1.[Calico 使用指南：Route Reflectors](https://docs.projectcalico.org/v3.3/usage/routereflector) +- 2.[BGP路由反射器基础](https://www.sohu.com/a/140033025_761420) + +更多 BGP 路由协议相关知识请查阅思科/华为相关网络文档。 diff --git a/docs/setup/network-plugin/calico.md b/docs/setup/network-plugin/calico.md new file mode 100644 index 0000000..6e1e360 --- /dev/null +++ b/docs/setup/network-plugin/calico.md @@ -0,0 +1,166 @@ +## 06-安装calico网络组件.md + +推荐阅读[calico kubernetes guide](https://docs.projectcalico.org/v3.0/getting-started/kubernetes/) + +本项目提供多种网络插件可选，如果需要安装calico，请在/etc/ansible/hosts文件中设置变量 `CLUSTER_NETWORK="calico"`，更多的calico设置在`roles/calico/defaults/main.yml`文件定义。 + +- calico-node需要在所有master节点和node节点安装 + +``` bash +roles/calico/ +├── tasks +│   └── main.yml +└── templates + ├── calico-csr.json.j2 + ├── calicoctl.cfg.j2 + └── calico.yaml.j2 +``` +请在另外窗口打开[roles/calico/tasks/main.yml](../roles/calico/tasks/main.yml) 文件，对照看以下讲解内容。 + +### 创建calico 证书申请 + +``` bash +{ + "CN": "calico", + "hosts": [], + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "CN", + "ST": "HangZhou", + "L": "XS", + "O": "k8s", + "OU": "System" + } + ] +} +``` +- calico 使用客户端证书，所以hosts字段可以为空；后续可以看到calico证书用在四个地方： + - calico/node 这个docker 容器运行时访问 etcd 使用证书 + - cni 配置文件中，cni 插件需要访问 etcd 使用证书 + - calicoctl 操作集群网络时访问 etcd 使用证书 + - calico/kube-controllers 同步集群网络策略时访问 etcd 使用证书 + +### 创建 calico DaemonSet yaml文件和rbac 文件 + +请对照 roles/calico/templates/calico.yaml.j2文件注释和以下注意内容 + ++ 详细配置参数请参考[calico官方文档](https://docs.projectcalico.org/v2.6/reference/node/configuration) ++ calico-node是以docker容器运行在host上的，因此需要把之前的证书目录 /etc/calico/ssl挂载到容器中 ++ 配置ETCD_ENDPOINTS 、CA、证书等，所有{{ }}变量与ansible hosts文件中设置对应 ++ 配置集群POD网络 CALICO_IPV4POOL_CIDR={{ CLUSTER_CIDR }} ++ **重要**本K8S集群运行在同网段kvm虚机上，虚机间没有网络ACL限制，因此可以设置`CALICO_IPV4POOL_IPIP=off`，如果你的主机位于不同网段，或者运行在公有云上需要打开这个选项 `CALICO_IPV4POOL_IPIP=always` ++ 配置FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT 默认允许Pod到Node的网络流量，更多[felix配置选项](https://docs.projectcalico.org/v2.6/reference/felix/configuration) ++ 多网卡服务器运行kube-controller可能会出现问题，本项目已通过增加设置环境变量`KUBERNETES_SERVICE_HOST`和`KUBERNETES_SERVICE_PORT`解决 + +### 安装calico 网络 + ++ 安装前检查主机名不能有大写字母，只能由`小写字母` `-` `.` 组成 (name must consist of lower case alphanumeric characters, '-' or '.' (regex: [a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*))(calico-node v3.0.6以上已经解决主机大写字母问题) ++ **安装前必须确保各节点主机名不重复** ，calico node name 由节点主机名决定，如果重复，那么重复节点在etcd中只存储一份配置，BGP 邻居也不会建立。 ++ 安装之前必须确保`kube-master`和`kube-node`节点已经成功部署 ++ 只需要在任意装有kubectl客户端的节点运行 `kubectl create `安装即可 ++ 等待15s后(视网络拉取calico相关镜像速度)，calico 网络插件安装完成，删除之前kube-node安装时默认cni网络配置 + +### [可选]配置calicoctl工具 [calicoctl.cfg.j2](roles/calico/templates/calicoctl.cfg.j2) + +``` bash +apiVersion: v1 +kind: calicoApiConfig +metadata: +spec: + datastoreType: "etcdv2" + etcdEndpoints: {{ ETCD_ENDPOINTS }} + etcdKeyFile: /etc/calico/ssl/calico-key.pem + etcdCertFile: /etc/calico/ssl/calico.pem + etcdCACertFile: /etc/calico/ssl/ca.pem +``` + +### 验证calico网络 + +执行calico安装成功后可以验证如下：(需要等待镜像下载完成，有时候即便上一步已经配置了docker国内加速，还是可能比较慢，请确认以下容器运行起来以后，再执行后续验证步骤) + +``` bash +kubectl get pod --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system calico-kube-controllers-5c6b98d9df-xj2n4 1/1 Running 0 1m +kube-system calico-node-4hr52 2/2 Running 0 1m +kube-system calico-node-8ctc2 2/2 Running 0 1m +kube-system calico-node-9t8md 2/2 Running 0 1m +``` + +**查看网卡和路由信息** + +先在集群创建几个测试pod: `kubectl run test --image=busybox --replicas=3 sleep 30000` + +``` bash +# 查看网卡信息 +ip a +``` + ++ 可以看到包含类似cali1cxxx的网卡，是calico为测试pod生成的 ++ tunl0网卡现在不用管，是默认生成的，当开启IPIP 特性时使用的隧道 + +``` bash +# 查看路由 +route -n +Kernel IP routing table +Destination Gateway Genmask Flags Metric Ref Use Iface +0.0.0.0 192.168.1.1 0.0.0.0 UG 0 0 0 ens3 +192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 ens3 +172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 +172.20.3.64 192.168.1.34 255.255.255.192 UG 0 0 0 ens3 +172.20.33.128 0.0.0.0 255.255.255.192 U 0 0 0 * +172.20.33.129 0.0.0.0 255.255.255.255 UH 0 0 0 caliccc295a6d4f +172.20.104.0 192.168.1.35 255.255.255.192 UG 0 0 0 ens3 +172.20.166.128 192.168.1.63 255.255.255.192 UG 0 0 0 ens3 +``` + +**查看所有calico节点状态** + +``` bash +calicoctl node status +Calico process is running. + +IPv4 BGP status ++--------------+-------------------+-------+----------+-------------+ +| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | ++--------------+-------------------+-------+----------+-------------+ +| 192.168.1.34 | node-to-node mesh | up | 12:34:00 | Established | +| 192.168.1.35 | node-to-node mesh | up | 12:34:00 | Established | +| 192.168.1.63 | node-to-node mesh | up | 12:34:01 | Established | ++--------------+-------------------+-------+----------+-------------+ +``` + +**BGP 协议是通过TCP 连接来建立邻居的，因此可以用netstat 命令验证 BGP Peer** + +``` bash +netstat -antlp|grep ESTABLISHED|grep 179 +tcp 0 0 192.168.1.66:179 192.168.1.35:41316 ESTABLISHED 28479/bird +tcp 0 0 192.168.1.66:179 192.168.1.34:40243 ESTABLISHED 28479/bird +tcp 0 0 192.168.1.66:179 192.168.1.63:48979 ESTABLISHED 28479/bird +``` + +**查看etcd中calico相关信息** + +因为这里calico网络使用etcd存储数据，所以可以在etcd集群中查看数据 + ++ calico 3.x 版本默认使用 etcd v3存储，**登陆集群的一个etcd 节点**，查看命令： + +``` bash +# 查看所有calico相关数据 +ETCDCTL_API=3 etcdctl --endpoints="http://127.0.0.1:2379" get --prefix /calico +# 查看 calico网络为各节点分配的网段 +ETCDCTL_API=3 etcdctl --endpoints="http://127.0.0.1:2379" get --prefix /calico/ipam/v2/host +``` + ++ calico 2.x 版本默认使用 etcd v2存储，**登陆集群的一个etcd 节点**，查看命令： + +``` bash +# 查看所有calico相关数据 +etcdctl --endpoints=http://127.0.0.1:2379 --ca-file=/etc/kubernetes/ssl/ca.pem ls /calico +``` + +## 下一步：[设置 BGP Route Reflector](calico-bgp-rr.md) diff --git a/docs/setup/network-plugin/cilium.md b/docs/setup/network-plugin/cilium.md new file mode 100644 index 0000000..257a442 --- /dev/null +++ b/docs/setup/network-plugin/cilium.md @@ -0,0 +1,224 @@ +# 06-安装cilium网络组件 + +`cilium` 是一个革新的网络与安全组件；基于 linux 内核新技术--`BPF`，它可以透明、零侵入地实现服务间安全策略与可视化，主要优势如下： + +- 支持L3/L4, L7(如：HTTP/gRPC/Kafka)的安全策略 +- 支持基于安全ID而不是地址+端口的传统防火墙策略 +- 支持基于Overlay或Native Routing的扁平多节点pod网络 + - Overlay VXLAN 方式类似于 flannel 的VXLAN后端 +- 高性能负载均衡，支持DSR +- 支持事件、策略跟踪和监控集成 + +## 开始使用 cilium + +以下为简要翻译 `cilium doc`上的一个应用示例[原文](http://docs.cilium.io/en/stable/gettingstarted/minikube/#step-2-deploy-the-demo-application)，部署在单节点k8s 环境的实践。 + +### 0.升级内核并重启 + +- Linux kernel >= 4.9.17，请阅读文档[升级内核](guide/kernel_upgrade.md) +- etcd >= 3.1.0 or consul >= 0.6.4 + +### 1.选择cilium网络后安装k8s(allinone) + +- 参考[快速指南](quickStart.md)，设置 ansible hosts 文件中变量 `CLUSTER_NETWORK="cilium"` + +### 2.部署示例应用 + +官方文档用几个`pod/svc` 抽象一个有趣的应用场景（星战迷）：星战中帝国方建造了被称为“终极武器”的“死星”，它是一个卫星大小的战斗空间站，它的核心是使用凯伯晶体（Kyber Crystal）的超级激光炮，剧中它的首秀就以完全火力摧毁了“杰达圣城”（Jedha）。下面将用运行于 k8s上的 pod/svc/cilium 等模拟“死星“的一个“飞船登陆”系统安全策略设计。 + +- deploy/deathstar：作为控制整个“死星”的飞船登陆管理系统，它暴露一个SVC，提供HTTP REST 接口给飞船请求登陆使用； +- pod/tiefighter：作为“帝国”方的常规战斗飞船，它会调用上述 HTTP 接口，请求登陆“死星”； +- pod/xwing：作为“盟军”方的飞行舰，它也尝试调用 HTTP 接口，请求登陆“死星”； + +![cilium_http_gsg](../pics/cilium_http_gsg.jpg) + +根据文件[http-sw-app.yaml](../roles/cilium/files/star_war_example/http-sw-app.yaml) 创建 `$ kubectl create -f http-sw-app.yaml` 后，验证如下： + +``` bash +$ kubectl get pods,svc +NAME READY STATUS RESTARTS AGE +pod/deathstar-5fc7c7795d-djf2q 1/1 Running 0 4h +pod/deathstar-5fc7c7795d-hrgst 1/1 Running 0 4h +pod/tiefighter 1/1 Running 0 4h +pod/xwing 1/1 Running 0 4h + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/deathstar ClusterIP 10.68.242.130 80/TCP 4h +service/kubernetes ClusterIP 10.68.0.1 443/TCP 5h +``` +每个 POD 在 `cilium` 中都表示为 `Endpoint`，初始每个 `Endpoint` 的”进出安全策略“状态均为 `Disabled`，如下：(已省略部分无关 POD 信息) + +``` bash +$ kubectl exec -n kube-system cilium-6t5vx -- cilium endpoint list +ENDPOINT POLICY (ingress) POLICY (egress) IDENTITY LABELS (source:key[=value]) IPv6 IPv4 STATUS + ENFORCEMENT ENFORCEMENT +643 Disabled Disabled 31371 k8s:class=deathstar f00d::ac14:0:0:283 172.20.0.246 ready + k8s:io.cilium.k8s.policy.serviceaccount=default + k8s:io.kubernetes.pod.namespace=default + k8s:org=empire +1011 Disabled Disabled 31371 k8s:class=deathstar f00d::ac14:0:0:3f3 172.20.0.63 ready + k8s:io.cilium.k8s.policy.serviceaccount=default + k8s:io.kubernetes.pod.namespace=default + k8s:org=empire +32030 Disabled Disabled 5350 k8s:class=tiefighter f00d::ac14:0:0:7d1e 172.20.0.201 ready + k8s:io.cilium.k8s.policy.serviceaccount=default + k8s:io.kubernetes.pod.namespace=default + k8s:org=empire +45943 Disabled Disabled 14309 k8s:class=xwing f00d::ac14:0:0:b377 172.20.0.189 ready + k8s:io.cilium.k8s.policy.serviceaccount=default + k8s:io.kubernetes.pod.namespace=default + k8s:org=alliance +52035 Disabled Disabled 4 reserved:health f00d::ac14:0:0:cb43 172.20.0.92 ready +``` + +### 3.检查初始状态 + +当然“死星”应该只允许“帝国”的飞船着陆，因为没有应用任何策略，所以初始状态下“帝国”和“联盟”的飞船都可以登陆，如下测试： + +``` bash +$ kubectl exec xwing -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing +Ship landed # 成功着陆 +$ kubectl exec tiefighter -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing +Ship landed # 成功着陆 +``` + +### 4.应用 L3/L4 策略 + +现在我们应用策略，仅让带有标签 `org=empire`的飞船登陆“死星”；那么带有标签 `org=alliance`的“联盟”飞船将禁止登陆；这个就是我们熟悉的传统L3/L4 防火墙策略，并跟踪连接（会话）状态； + +![cilium_http_l3_l4_gsg](../pics/cilium_http_l3_l4_gsg.jpg) + +根据文件[sw_l3_l4_policy.yaml](../roles/cilium/files/star_war_example/sw_l3_l4_policy.yaml) 创建 `$ kubectl apply -f sw_l3_l4_policy.yaml` 后，验证如下： + +``` bash +$ kubectl exec tiefighter -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing +Ship landed # 成功着陆 + +$ kubectl exec xwing -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing +# 失败超时 +``` + +### 5.查看安全策略 + +再次执行 `cilium endpoint list`，可以看到标签带`deathstar`的 POD 已经应用了 `Ingress`方向的策略： + +``` bash +# kubectl exec -n kube-system cilium-6t5vx -- cilium endpoint list +ENDPOINT POLICY (ingress) POLICY (egress) IDENTITY LABELS (source:key[=value]) IPv6 IPv4 STATUS + ENFORCEMENT ENFORCEMENT +643 Enabled Disabled 31371 k8s:class=deathstar f00d::ac14:0:0:283 172.20.0.246 ready + k8s:io.cilium.k8s.policy.serviceaccount=default + k8s:io.kubernetes.pod.namespace=default + k8s:org=empire +1011 Enabled Disabled 31371 k8s:class=deathstar f00d::ac14:0:0:3f3 172.20.0.63 ready + k8s:io.cilium.k8s.policy.serviceaccount=default + k8s:io.kubernetes.pod.namespace=default + k8s:org=empire +32030 Disabled Disabled 5350 k8s:class=tiefighter f00d::ac14:0:0:7d1e 172.20.0.201 ready + k8s:io.cilium.k8s.policy.serviceaccount=default + k8s:io.kubernetes.pod.namespace=default + k8s:org=empire +45943 Disabled Disabled 14309 k8s:class=xwing f00d::ac14:0:0:b377 172.20.0.189 ready + k8s:io.cilium.k8s.policy.serviceaccount=default + k8s:io.kubernetes.pod.namespace=default + k8s:org=alliance +52035 Disabled Disabled 4 reserved:health f00d::ac14:0:0:cb43 172.20.0.92 ready +``` + +查看具体策略内容 `kubectl describe cnp rule1` + +### 6. L7 安全策略 + +上述的策略可以进行简单的安全防护了，但是“死星”的这个系统还有很多复杂的功能；比如它还提供了一个内部维护接口，如果被不合理调用将带来严重灾难性后果，也许“联盟”勇士劫持了一架“帝国”飞船正在进行这个任务（虽然我们内心希望他能够成功摧毁“死星”）。不幸的是“死星”系统设计者考虑到这个风险，它有办法严格限制每架飞船能够请求的权限。 + +没有限制飞船请求权限时，如下运行： + +``` bash +$ kubectl exec tiefighter -- curl -s -XPUT deathstar.default.svc.cluster.local/v1/exhaust-port +Panic: deathstar exploded + +goroutine 1 [running]: +main.HandleGarbage(0x2080c3f50, 0x2, 0x4, 0x425c0, 0x5, 0xa) + /code/src/github.com/empire/deathstar/ + temp/main.go:9 +0x64 +main.main() + /code/src/github.com/empire/deathstar/ + temp/main.go:5 +0x85 +``` + +![cilium_http_l3_l4_l7_gsg](../pics/cilium_http_l3_l4_l7_gsg.jpg) + +限制L7 的安全策略，根据文件[sw_l3_l4_l7_policy.yaml](../roles/cilium/files/star_war_example/sw_l3_l4_l7_policy.yaml) 创建 `$ kubectl apply -f sw_l3_l4_l7_policy.yaml` 后，验证如下： + +``` bash +$ kubectl exec tiefighter -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing +Ship landed +$ kubectl exec tiefighter -- curl -s -XPUT deathstar.default.svc.cluster.local/v1/exhaust-port +Access denied +``` + +我们同样可以使用 `kubectl desribe cnp`检查更新的策略，或者使用 `cilium` 命令行： + +``` bash +$ kubectl exec -n kube-system cilium-6t5vx -- cilium policy get +[ + { + "endpointSelector": { + "matchLabels": { + "any:class": "deathstar", + "any:org": "empire", + "k8s:io.kubernetes.pod.namespace": "default" + } + }, + "ingress": [ + { + "fromEndpoints": [ + { + "matchLabels": { + "any:org": "empire", + "k8s:io.kubernetes.pod.namespace": "default" + } + } + ], + "toPorts": [ + { + "ports": [ + { + "port": "80", + "protocol": "TCP" + } + ], + "rules": { + "http": [ + { + "path": "/v1/request-landing", + "method": "POST" + } + ] + } + } + ] + } + ], + "labels": [ + { + "key": "io.cilium.k8s.policy.name", + "value": "rule1", + "source": "k8s" + }, + { + "key": "io.cilium.k8s.policy.namespace", + "value": "default", + "source": "k8s" + } + ] + } +] +Revision: 267 +``` +我们看到 `cilium` 可以实现 `7层 HTTP `协议的请求方法（GET/PUT/POST等）、路径（/v1/request-landing）等等安全策略；另外，它还可以防护其他应用（如：Kafka, gRPC, Elasticsearch），可以去官网文档示例学习！ + +## 参考资料 + +- [cilium github](https://github.com/cilium/cilium) +- [cilium doc](http://docs.cilium.io) diff --git a/docs/setup/network-plugin/flannel.md b/docs/setup/network-plugin/flannel.md new file mode 100644 index 0000000..4148e92 --- /dev/null +++ b/docs/setup/network-plugin/flannel.md @@ -0,0 +1,126 @@ +## 06-安装flannel网络组件.md + +本项目提供多种网络插件可选，如果需要安装flannel，请在/etc/ansible/hosts文件中设置变量 `CLUSTER_NETWORK="flannel"`，更多设置请查看`roles/flannel/defaults/main.yml` + +`Flannel`是最早应用到k8s集群的网络插件之一，简单高效，且提供多个后端`backend`模式供选择；本文介绍以`DaemonSet Pod`方式集成到k8s集群，需要在所有master节点和node节点安装。 + +``` text +roles/flannel/ +├── tasks +│   └── main.yml +└── templates + └── kube-flannel.yaml.j2 +``` + +请在另外窗口打开[roles/flannel/tasks/main.yml](../roles/flannel/tasks/main.yml) 文件，对照看以下讲解内容。 + +### 下载基础cni 插件 + +请到CNI 插件最新[release](https://github.com/containernetworking/plugins/releases)页面下载[cni-v0.6.0.tgz](https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-v0.6.0.tgz)，解压后里面有很多插件，选择如下几个复制到项目 `bin`目录下 + +- flannel用到的插件 + - bridge + - flannel + - host-local + - loopback + - portmap + +Flannel CNI 插件的配置文件可以包含多个`plugin` 或由其调用其他`plugin`；`Flannel DaemonSet Pod`运行以后会生成`/run/flannel/subnet.env `文件，例如： + +``` bash +FLANNEL_NETWORK=10.1.0.0/16 +FLANNEL_SUBNET=10.1.17.1/24 +FLANNEL_MTU=1472 +FLANNEL_IPMASQ=true +``` +然后它利用这个文件信息去配置和调用`bridge`插件来生成容器网络，调用`host-local`来管理`IP`地址，例如： + +``` bash +{ + "name": "mynet", + "type": "bridge", + "mtu": 1472, + "ipMasq": false, + "isGateway": true, + "ipam": { + "type": "host-local", + "subnet": "10.1.17.0/24" + } +} +``` +- 更多相关介绍请阅读： + - [flannel kubernetes 集成](https://github.com/coreos/flannel/blob/master/Documentation/kubernetes.md) + - [flannel cni 插件](https://github.com/containernetworking/plugins/tree/master/plugins/meta/flannel) + - [更多 cni 插件](https://github.com/containernetworking/plugins) + +### 准备`Flannel DaemonSet` yaml配置文件 + +请阅读 `roles/flannel/templates/kube-flannel.yaml.j2` 内容，注意： + ++ 本安装方式，flannel使用apiserver 存储数据 ++ 配置相关RBAC 权限和 `service account` ++ 配置`ConfigMap`包含 CNI配置和 flannel配置(指定backend等)，和`hosts`文件中相关设置对应 ++ `DaemonSet Pod`包含两个容器，一个容器运行flannel本身，另一个init容器部署cni 配置文件 ++ 为方便国内加速使用镜像 `jmgao1983/flannel:v0.10.0-amd64` (官方镜像在docker-hub上的转存) ++ 特别注意：如果服务器是多网卡（例如vagrant环境），则需要在`roles/flannel/templates/kube-flannel.yaml.j2 `中增加指定环境变量，详见 [kubernetes ISSUE 39701](https://github.com/kubernetes/kubernetes/issues/39701) + +``` bash + ... + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST # 指定apiserver的主机地址 + value: {{ MASTER_IP }} + - name: KUBERNETES_SERVICE_PORT # 指定apiserver的服务端口 + value: {{ KUBE_APISERVER.split(':')[2] }} + ... +``` +### 安装 flannel网络 + ++ 安装之前必须确保kube-master和kube-node节点已经成功部署 ++ 只需要在任意装有kubectl客户端的节点运行 kubectl create安装即可 ++ 等待15s后(视网络拉取相关镜像速度)，flannel 网络插件安装完成，删除之前kube-node安装时默认cni网络配置 + +### 验证flannel网络 + +执行flannel安装成功后可以验证如下：(需要等待镜像下载完成，有时候即便上一步已经配置了docker国内加速，还是可能比较慢，请确认以下容器运行起来以后，再执行后续验证步骤) + +``` bash +# kubectl get pod --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system kube-flannel-ds-m8mzm 1/1 Running 0 3m +kube-system kube-flannel-ds-mnj6j 1/1 Running 0 3m +kube-system kube-flannel-ds-mxn6k 1/1 Running 0 3m +``` +在集群创建几个测试pod: `kubectl run test --image=busybox --replicas=3 sleep 30000` + +``` bash +# kubectl get pod --all-namespaces -o wide|head -n 4 +NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE +default busy-5956b54c8b-ld4gb 1/1 Running 0 9m 172.20.2.7 192.168.1.1 +default busy-5956b54c8b-lj9l9 1/1 Running 0 9m 172.20.1.5 192.168.1.2 +default busy-5956b54c8b-wwpkz 1/1 Running 0 9m 172.20.0.6 192.168.1.3 + +# 查看路由 +# ip route +default via 192.168.1.254 dev ens3 onlink +192.168.1.0/24 dev ens3 proto kernel scope link src 192.168.1.1 +172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown +172.20.0.0/24 via 192.168.1.3 dev ens3 +172.20.1.0/24 via 192.168.1.2 dev ens3 +172.20.2.0/24 dev cni0 proto kernel scope link src 172.20.2.1 +``` +在各节点上分别 ping 这三个POD IP地址，确保能通： + +``` bash +ping 172.20.2.7 +ping 172.20.1.5 +ping 172.20.0.6 +``` + diff --git a/docs/setup/network-plugin/kube-router.md b/docs/setup/network-plugin/kube-router.md new file mode 100644 index 0000000..6d28c1d --- /dev/null +++ b/docs/setup/network-plugin/kube-router.md @@ -0,0 +1,107 @@ +# kube-router 网络组件 + +kube-router是一个简单、高效的网络插件，它提供一揽子解决方案： +- 基于GoBGP 提供Pod 网络互联（Routing） +- 使用ipsets优化的iptables 提供网络策略支持（Firewall/NetworkPolicy） +- 基于IPVS/LVS 提供高性能服务代理（Service Proxy） + +更多介绍请前往`https://github.com/cloudnativelabs/kube-router` + +## 配置 + +本项目提供多种网络插件可选，如果需要安装kube-router，请在/etc/ansible/hosts文件中设置变量 `CLUSTER_NETWORK="kube-router"`，更多设置请查看`roles/kube-router/defaults/main.yml` + +- kube-router需要在所有master节点和node节点安装 + +## 安装 + +- 单步安装已经集成：`ansible-playbook 90.setup.yml` +- 分步安装请执行：`ansible-playbook 06.network.yml` + +## 验证 + +- 1.pod间网络联通性：略 + +- 2.host路由表 + +``` bash +# master上路由 +root@master1:~$ ip route +... +172.20.1.0/24 via 192.168.1.2 dev ens3 proto 17 +172.20.2.0/24 via 192.168.1.3 dev ens3 proto 17 +... + +# node3上路由 +root@node3:~$ ip route +... +172.20.0.0/24 via 192.168.1.1 dev ens3 proto 17 +172.20.1.0/24 via 192.168.1.2 dev ens3 proto 17 +172.20.2.0/24 dev kube-bridge proto kernel scope link src 172.20.2.1 +... +``` + +- 3.bgp连接状态 + +``` bash +# master上 +root@master1:~$ netstat -antlp|grep router|grep LISH|grep 179 +tcp 0 0 192.168.1.1:179 192.168.1.3:58366 ESTABLISHED 26062/kube-router +tcp 0 0 192.168.1.1:42537 192.168.1.2:179 ESTABLISHED 26062/kube-router + +# node3上 +root@node3:~$ netstat -antlp|grep router|grep LISH|grep 179 +tcp 0 0 192.168.1.3:58366 192.168.1.1:179 ESTABLISHED 18897/kube-router +tcp 0 0 192.168.1.3:179 192.168.1.2:43928 ESTABLISHED 18897/kube-router + +``` + +- 4.NetworkPolicy有效性，验证参照[这里](guide/networkpolicy.md) + +- 5.ipset列表查看 + +``` bash +$ ipset list +... +Name: kube-router-pod-subnets +Type: hash:net +Revision: 6 +Header: family inet hashsize 1024 maxelem 65536 timeout 0 +Size in memory: 672 +References: 2 +Members: +172.20.1.0/24 timeout 0 +172.20.2.0/24 timeout 0 +172.20.0.0/24 timeout 0 + +Name: kube-router-node-ips +Type: hash:ip +Revision: 4 +Header: family inet hashsize 1024 maxelem 65536 timeout 0 +Size in memory: 416 +References: 1 +Members: +192.168.1.1 timeout 0 +192.168.1.2 timeout 0 +192.168.1.3 timeout 0 +... +``` + +- 6.ipvs虚拟服务器查看 (roles/kube-router/defaults/main.yml 需配置`SERVICE_PROXY: "true"`) + +``` bash +# 首先创建测试应用 +$ kubectl run nginx --image=nginx --replicas=3 --port=80 --expose + +# 查看ipvsadm输出 +$ ipvsadm +IP Virtual Server version 1.2.1 (size=4096) +Prot LocalAddress:Port Scheduler Flags + -> RemoteAddress:Port Forward Weight ActiveConn InActConn +TCP 10.68.0.1:https rr persistent 10800 # 这个kubernetes虚拟服务地址 + -> 192.168.1.1:6443 Masq 1 0 0 +TCP 10.68.199.39:http rr # 这个是测试应用nginx的虚拟服务地址 + -> 172.20.1.5:http Masq 1 0 0 + -> 172.20.2.6:http Masq 1 0 0 + -> 172.20.2.8:http Masq 1 0 0 +``` diff --git a/docs/setup/quickStart.md b/docs/setup/quickStart.md new file mode 100644 index 0000000..c72e4e0 --- /dev/null +++ b/docs/setup/quickStart.md @@ -0,0 +1,147 @@ +## 快速指南 + +以下为快速体验k8s集群的测试、开发环境--allinone部署，国内环境下觉得比官方的minikube方便、简单很多。 + +### 1.基础系统配置 + ++ 推荐内存2G/硬盘30G以上 ++ 最小化安装`Ubuntu 16.04 server`或者`CentOS 7 Minimal` ++ 配置基础网络、更新源、SSH登陆等 + +### 2.安装依赖工具 + +Ubuntu 16.04 请执行以下脚本: + +``` bash +# 文档中脚本默认均以root用户执行 +# 安装依赖工具 +apt-get install python2.7 git python-pip +# Ubuntu16.04可能需要配置以下软连接 +ln -s /usr/bin/python2.7 /usr/bin/python +``` +CentOS 7 请执行以下脚本： + +``` bash +# 文档中脚本默认均以root用户执行 +# 安装 epel 源 +yum install epel-release -y +# 安装依赖工具 +yum install git python python-pip -y +``` +### 3.ansible安装及准备 + +``` bash +# 安装ansible (国内如果安装太慢可以直接用pip阿里云加速) +#pip install pip --upgrade +#pip install ansible +pip install pip --upgrade -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +pip install --no-cache-dir ansible -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com +# 配置ansible ssh密钥登陆 +ssh-keygen -t rsa -b 2048 回车 回车 回车 +ssh-copy-id $IP #$IP为本虚机地址，按照提示输入yes 和root密码 +``` + +在`Ubuntu 16.04`中，如果出现以下错误: + +``` bash +Traceback (most recent call last): + File "/usr/bin/pip", line 9, in + from pip import main +ImportError: cannot import name main +``` +将`/usr/bin/pip`做以下修改： + +``` bash +#原代码 +from pip import main +if __name__ == '__main__': + sys.exit(main()) + +#修改后 +from pip import __main__ +if __name__ == '__main__': + sys.exit(__main__._main()) +``` + +### 4.安装kubernetes集群 + +- 4.1 下载项目源码 + +``` bash +# 方式一：使用git clone +git clone https://github.com/gjmzj/kubeasz.git +mkdir -p /etc/ansible +mv kubeasz/* /etc/ansible +# 方式二：从发布页面 https://github.com/gjmzj/kubeasz/releases 下载源码解压到同样目录 +``` +- 4.2a 下载二进制文件 +请从分享的[百度云链接](https://pan.baidu.com/s/1c4RFaA)，下载解压到/etc/ansible/bin目录，如果你有合适网络环境也可以按照/down/download.sh自行从官网下载各种tar包 + +``` bash +tar zxvf k8s.1-9-8.tar.gz # 以安装k8s v1.9.8为例 +mv bin/* /etc/ansible/bin +``` +- 4.2b [可选]下载离线docker镜像 +服务器使用内部yum源/apt源，但是无法访问公网情况下，请下载离线docker镜像完成集群安装；从百度云盘把`basic_images_kubeasz_x.y.tar.gz` 下载解压到`/etc/ansible/down` 目录 + +``` bash +tar zxvf basic_images_kubeasz_0.2.tar.gz -C /etc/ansible/down +``` +- 4.3 配置集群参数 + - 4.3.1 必要配置：`cd /etc/ansible && cp example/hosts.allinone.example hosts`, 然后实际情况修改此hosts文件 + - 4.3.2 可选配置，初次使用可以不做修改，详见[配置指南](config_guide.md) + - 4.3.3 验证ansible 安装：`ansible all -m ping` 正常能看到节点返回 SUCCESS + +- 4.4 开始安装 +如果你对集群安装流程不熟悉，请阅读项目首页 **安装步骤** 讲解后分步安装，并对 **每步都进行验证** + +``` bash +# 分步安装 +ansible-playbook 01.prepare.yml +ansible-playbook 02.etcd.yml +ansible-playbook 03.docker.yml +ansible-playbook 04.kube-master.yml +ansible-playbook 05.kube-node.yml +ansible-playbook 06.network.yml +ansible-playbook 07.cluster-addon.yml +# 一步安装 +#ansible-playbook 90.setup.yml +``` + ++ [可选]对集群节点进行操作系统层面的安全加固 `ansible-playbook roles/os-harden/os-harden.yml`，详情请参考[os-harden项目](https://github.com/dev-sec/ansible-os-hardening) + +### 5.验证安装 +如果提示kubectl: command not found，退出重新ssh登陆一下，环境变量生效即可 + +``` bash +kubectl version +kubectl get componentstatus # 可以看到scheduler/controller-manager/etcd等组件 Healthy +kubectl cluster-info # 可以看到kubernetes master(apiserver)组件 running +kubectl get node # 可以看到单 node Ready状态 +kubectl get pod --all-namespaces # 可以查看所有集群pod状态，默认已安装网络插件、coredns、metrics-server等 +kubectl get svc --all-namespaces # 可以查看所有集群服务状态 +``` +### 6.安装主要组件 + +``` bash +# 安装kubedns，默认已集成安装 +#kubectl create -f /etc/ansible/manifests/kubedns +# 安装dashboard，默认已集成安装 +#kubectl create -f /etc/ansible/manifests/dashboard +``` ++ 登陆 `dashboard`可以查看和管理集群，更多内容请查阅[dashboard文档](../guide/dashboard.md) + +### 7.清理集群 + +以上步骤创建的K8S开发测试环境请尽情折腾，碰到错误尽量通过查看日志、上网搜索、提交`issues`等方式解决；当然如果是彻底奔溃了，可以清理集群后重新创建。 + +``` bash +ansible-playbook 99.clean.yml +``` + +如果出现清理失败，类似报错：`... Device or resource busy: '/var/run/docker/netns/xxxxxxxxxx'`，需要手动umount该目录后清理 + +``` bash +$ umount /var/run/docker/netns/xxxxxxxxxx +$ rm -rf /var/run/docker/netns/xxxxxxxxxx +``` diff --git a/down/download.sh b/down/download.sh new file mode 100644 index 0000000..1553080 --- /dev/null +++ b/down/download.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# This script describes where to download the official released binaries needed +# It's suggested to download the entire *.tar.gz at https://pan.baidu.com/s/1c4RFaA + +# example releases +K8S_VER=v1.10.4 +ETCD_VER=v3.3.8 +DOCKER_VER=17.03.2-ce +CNI_VER=v0.6.0 +DOCKER_COMPOSE=1.18.0 +HARBOR=v1.5.2 + +echo "\nNote1: Before this script, please finish downloading binaries manually from following urls." +echo "\nNote2：If binaries are not ready, use `Ctrl + C` to stop this script." + +echo "\n----download k8s binary at:" +echo https://dl.k8s.io/${K8S_VER}/kubernetes-server-linux-amd64.tar.gz + +echo "\n----download etcd binary at:" +echo https://github.com/coreos/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz +echo https://storage.googleapis.com/etcd/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz + +echo "\n----download docker binary at:" +echo https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VER}.tgz + +echo "\n----download ca tools at:" +echo https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 +echo https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 +echo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 + +echo "\n----download docker-compose at:" +echo https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE}/docker-compose-Linux-x86_64 + +echo "\n----download harbor-offline-installer at:" +echo https://github.com/vmware/harbor/releases/download/${HARBOR}/harbor-offline-installer-${HARBOR}.tgz + +echo "\n----download cni plugins at:" +echo https://github.com/containernetworking/plugins/releases + +sleep 30 + +### prepare 'cfssl' cert tool suit +echo "\nMoving 'cfssl' to 'bin' dir..." +if [ -f "cfssl_linux-amd64" ]; then + mv -f cfssl_linux-amd64 ../bin/cfssl +else + echo Please download 'cfssl' at 'https://pkg.cfssl.org/R1.2/cfssl_linux-amd64' +fi +if [ -f "cfssljson_linux-amd64" ]; then + mv -f cfssljson_linux-amd64 ../bin/cfssljson +else + echo Please download 'cfssljson' at 'https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64' +fi +if [ -f "cfssl-certinfo_linux-amd64" ]; then + mv -f cfssl-certinfo_linux-amd64 ../bin/cfssl-certinfo +else + echo Please download 'cfssl-certinfo' at 'https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64' +fi + +### prepare 'etcd' binaries +if [ -f "etcd-${ETCD_VER}-linux-amd64.tar.gz" ]; then + echo "\nextracting etcd binaries..." + tar zxf etcd-${ETCD_VER}-linux-amd64.tar.gz + mv -f etcd-${ETCD_VER}-linux-amd64/etcd* ../bin +else + echo Please download 'etcd-${ETCD_VER}-linux-amd64.tar.gz' first +fi + +### prepare kubernetes binaries +if [ -f "kubernetes-server-linux-amd64.tar.gz" ]; then + echo "\nextracting kubernetes binaries..." + tar zxf kubernetes-server-linux-amd64.tar.gz + mv -f kubernetes/server/bin/kube-apiserver ../bin + mv -f kubernetes/server/bin/kube-controller-manager ../bin + mv -f kubernetes/server/bin/kubectl ../bin + mv -f kubernetes/server/bin/kubelet ../bin + mv -f kubernetes/server/bin/kube-proxy ../bin + mv -f kubernetes/server/bin/kube-scheduler ../bin +else + echo Please download 'kubernetes-server-linux-amd64.tar.gz' first +fi + +### prepare docker binaries +if [ -f "docker-${DOCKER_VER}.tgz" ]; then + echo "\nextracting docker binaries..." + tar zxf docker-${DOCKER_VER}.tgz + mv -f docker/docker* ../bin + if [ -f "docker/completion/bash/docker" ]; then + mv -f docker/completion/bash/docker ../roles/docker/files/docker + fi +else + echo Please download 'docker-${DOCKER_VER}.tgz' first +fi + +### prepare cni plugins, needed by flannel; +if [ -f "cni-${CNI_VER}.tgz" ]; then + echo "\nextracting cni plugins binaries..." + tar zxf cni-${CNI_VER}.tgz + mv -f bridge ../bin + mv -f flannel ../bin + mv -f host-local ../bin + mv -f loopback ../bin + mv -f portmap ../bin +else + echo Please download 'cni-${CNI_VER}.tgz' first +fi diff --git a/down/offline_images b/down/offline_images new file mode 100644 index 0000000..c2d5ae6 --- /dev/null +++ b/down/offline_images @@ -0,0 +1,50 @@ +###---[basic_images ] +# dns-addon +coredns/coredns:1.2.6 +mirrorgooglecontainers/k8s-dns-kube-dns-amd64:1.14.13 +mirrorgooglecontainers/k8s-dns-dnsmasq-nanny-amd64:1.14.13 +mirrorgooglecontainers/k8s-dns-sidecar-amd64:1.14.13 +# metrics-server +mirrorgooglecontainers/metrics-server-amd64:v0.3.1 +# calico +calico/node:v3.4.1 +calico/cni:v3.4.1 +calico/kube-controllers:v3.4.1 +# cilium +cilium/cilium:v1.1.4 +# flannel +jmgao1983/flannel:v0.11.0-amd64 +# kube-router +cloudnativelabs/kube-router:v0.2.0 +# dashboard +mirrorgooglecontainers/kubernetes-dashboard-amd64:v1.10.0 +# pause +mirrorgooglecontainers/pause-amd64:3.1 +busybox:1.28.4 +# traefik ingress +traefik:v1.7.4 +# heapster +mirrorgooglecontainers/heapster-grafana-amd64:v4.4.3 +mirrorgooglecontainers/heapster-amd64:v1.5.4 +mirrorgooglecontainers/heapster-influxdb-amd64:v1.3.3 + +###---[extra-images] +# helm tiller server +jmgao1983/tiller:v2.12.3 +# efk +mirrorgooglecontainers/elasticsearch:v5.6.4 +alpine:3.6 +mirrorgooglecontainers/fluentd-elasticsearch:v2.0.2 +jmgao1983/kibana:5.6.4 +# nfs dynamic PV provisioner +jmgao1983/nfs-client-provisioner:latest +# prometheus +busybox:latest +grafana/grafana:5.1.2 +mirrorgooglecontainers/kube-state-metrics:v1.3.1 +appropriate/curl:latest +prom/prometheus:v2.2.1 +prom/alertmanager:v0.14.0 +prom/node-exporter:v0.15.2 +jimmidyson/configmap-reload:v0.1 + diff --git a/example/hosts.allinone.example b/example/hosts.allinone.example new file mode 100644 index 0000000..35249d2 --- /dev/null +++ b/example/hosts.allinone.example @@ -0,0 +1,68 @@ +# 集群部署节点：一般为运行ansible 脚本的节点 +# 变量 NTP_ENABLED (=yes/no) 设置集群是否安装 chrony 时间同步 +[deploy] +192.168.1.1 NTP_ENABLED=no + +# etcd集群请提供如下NODE_NAME，注意etcd集群必须是1,3,5,7...奇数个节点 +[etcd] +192.168.1.1 NODE_NAME=etcd1 + +[kube-master] +192.168.1.1 + +[kube-node] +192.168.1.1 + +# 参数 NEW_INSTALL：yes表示新建，no表示使用已有harbor服务器 +# 如果不使用域名，可以设置 HARBOR_DOMAIN="" +[harbor] +#192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no + +# 【可选】外部负载均衡，用于自有环境负载转发 NodePort 暴露的服务等 +[ex-lb] +#192.168.1.6 LB_ROLE=backup EX_VIP=192.168.1.250 +#192.168.1.7 LB_ROLE=master EX_VIP=192.168.1.250 + +[all:vars] +# ---------集群主要参数--------------- +#集群部署模式：allinone, single-master, multi-master +DEPLOY_MODE=allinone + +#集群 MASTER IP，自动生成 +MASTER_IP="{{ groups['kube-master'][0] }}" +KUBE_APISERVER="https://{{ MASTER_IP }}:6443" + +# 集群网络插件，目前支持calico, flannel, kube-router, cilium +CLUSTER_NETWORK="flannel" + +# 服务网段 (Service CIDR），注意不要与内网已有网段冲突 +SERVICE_CIDR="10.68.0.0/16" + +# POD 网段 (Cluster CIDR），注意不要与内网已有网段冲突 +CLUSTER_CIDR="172.20.0.0/16" + +# 服务端口范围 (NodePort Range) +NODE_PORT_RANGE="20000-40000" + +# kubernetes 服务 IP (预分配，一般是 SERVICE_CIDR 中第一个IP) +CLUSTER_KUBERNETES_SVC_IP="10.68.0.1" + +# 集群 DNS 服务 IP (从 SERVICE_CIDR 中预分配) +CLUSTER_DNS_SVC_IP="10.68.0.2" + +# 集群 DNS 域名 +CLUSTER_DNS_DOMAIN="cluster.local." + +# 集群basic auth 使用的用户名和密码 (运行时会生成随机密码) +BASIC_AUTH_USER="admin" +BASIC_AUTH_PASS="test1234" + +# ---------附加参数-------------------- +#默认二进制文件目录 +bin_dir="/opt/kube/bin" + +#证书目录 +ca_dir="/etc/kubernetes/ssl" + +#部署目录，即 ansible 工作目录 +base_dir="/etc/ansible" diff --git a/example/hosts.allinone.example.en b/example/hosts.allinone.example.en new file mode 100644 index 0000000..157d263 --- /dev/null +++ b/example/hosts.allinone.example.en @@ -0,0 +1,69 @@ +# 'deploy' node, which the ansible-playbooks usually run on +# variable 'NTP_ENABLED(=yes/no)' enables/disables the NTP server 'chrony' +[deploy] +192.168.1.1 NTP_ENABLED=no + +# 'etcd' cluster must have odd member(s) (1,3,5,...) +# variable 'NODE_NAME' is the distinct name of a member in 'etcd' cluster +[etcd] +192.168.1.1 NODE_NAME=etcd1 + +[kube-master] +192.168.1.1 + +[kube-node] +192.168.1.1 + +# set 'NEW_INSTALL': 'yes' to install a harbor server; 'no' to integrate with existed one +# if not using domian，set 'HARBOR_DOMAIN=""' +[harbor] +#192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no + +# [optional] loadbalance for services with type 'NodePort' +[ex-lb] +#192.168.1.6 LB_ROLE=backup EX_VIP=192.168.1.250 +#192.168.1.7 LB_ROLE=master EX_VIP=192.168.1.250 + +[all:vars] +# --------- Main Variables --------------- +# Cluster Deployment Mode: allinone, single-master, multi-master +DEPLOY_MODE=allinone + +# Cluster's Master IP, auto generated +MASTER_IP="{{ groups['kube-master'][0] }}" +KUBE_APISERVER="https://{{ MASTER_IP }}:6443" + +# Network plugins supported: calico, flannel, kube-router, cilium +CLUSTER_NETWORK="flannel" + +# K8S Service CIDR, not overlap with node(host) networking +SERVICE_CIDR="10.68.0.0/16" + +# Cluster CIDR (Pod CIDR), not overlap with node(host) networking +CLUSTER_CIDR="172.20.0.0/16" + +# NodePort Range +NODE_PORT_RANGE="20000-40000" + +# Kubernetes SVC IP (usually assigned with the first available IP of 'SERVICE_CIDR') +CLUSTER_KUBERNETES_SVC_IP="10.68.0.1" + +# Cluster DNS Server's IP (assigned with an available IP of 'SERVICE_CIDR') +CLUSTER_DNS_SVC_IP="10.68.0.2" + +# Cluster DNS Domain +CLUSTER_DNS_DOMAIN="cluster.local." + +# Basic auth for apiserver (a random password will be gennerated on cluster setup) +BASIC_AUTH_USER="admin" +BASIC_AUTH_PASS="test1234" + +# -------- Additional Variables -------------------- +# Binaries Directory +bin_dir="/opt/kube/bin" + +# CA and other components cert/key Directory +ca_dir="/etc/kubernetes/ssl" + +# Deploy Directory (kubeasz workspace), don't change the default value right now +base_dir="/etc/ansible" diff --git a/example/hosts.cloud.example b/example/hosts.cloud.example new file mode 100644 index 0000000..5a7804c --- /dev/null +++ b/example/hosts.cloud.example @@ -0,0 +1,68 @@ +# 集群部署节点：一般为运行ansible 脚本的节点 +# 变量 NTP_ENABLED (=yes/no) 设置集群是否安装 chrony 时间同步, 公有云上虚机不需要 +[deploy] +192.168.1.1 NTP_ENABLED=no + +# etcd集群请提供如下NODE_NAME，注意etcd集群必须是1,3,5,7...奇数个节点 +[etcd] +192.168.1.1 NODE_NAME=etcd1 +192.168.1.2 NODE_NAME=etcd2 +192.168.1.3 NODE_NAME=etcd3 + +[kube-master] +192.168.1.2 +192.168.1.3 + +[kube-node] +192.168.1.1 +192.168.1.4 + +# 参数 NEW_INSTALL：yes表示新建，no表示使用已有harbor服务器 +# 如果不使用域名，可以设置 HARBOR_DOMAIN="" +[harbor] +#192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no + +[all:vars] +# ---------集群主要参数--------------- +#集群部署模式：allinone, single-master, multi-master +DEPLOY_MODE=multi-master + +# 公有云上一般都有提供负载均衡产品，且不允许自己创建lb +# 创建公有云内网负载均衡，然后配置：前端监听 tcp 8443，后端 tcp 6443，后端节点即 master 节点 +MASTER_IP="192.168.1.10" # 即负载均衡内网地址 +KUBE_APISERVER="https://{{ MASTER_IP }}:8443" + +# 集群网络插件，目前支持calico, flannel, kube-router, cilium +CLUSTER_NETWORK="flannel" + +# 服务网段 (Service CIDR），注意不要与内网已有网段冲突 +SERVICE_CIDR="10.68.0.0/16" + +# POD 网段 (Cluster CIDR），注意不要与内网已有网段冲突 +CLUSTER_CIDR="172.20.0.0/16" + +# 服务端口范围 (NodePort Range) +NODE_PORT_RANGE="20000-40000" + +# kubernetes 服务 IP (预分配，一般是 SERVICE_CIDR 中第一个IP) +CLUSTER_KUBERNETES_SVC_IP="10.68.0.1" + +# 集群 DNS 服务 IP (从 SERVICE_CIDR 中预分配) +CLUSTER_DNS_SVC_IP="10.68.0.2" + +# 集群 DNS 域名 +CLUSTER_DNS_DOMAIN="cluster.local." + +# 集群basic auth 使用的用户名和密码 (运行时会生成随机密码) +BASIC_AUTH_USER="admin" +BASIC_AUTH_PASS="test1234" + +# ---------附加参数-------------------- +#默认二进制文件目录 +bin_dir="/opt/kube/bin" + +#证书目录 +ca_dir="/etc/kubernetes/ssl" + +#部署目录，即 ansible 工作目录，建议不要修改 +base_dir="/etc/ansible" diff --git a/example/hosts.cloud.example.en b/example/hosts.cloud.example.en new file mode 100644 index 0000000..67fa159 --- /dev/null +++ b/example/hosts.cloud.example.en @@ -0,0 +1,69 @@ +# 'deploy' node, which the ansible-playbooks usually run on +# variable 'NTP_ENABLED(=yes/no)' enables/disables the NTP server 'chrony' +[deploy] +192.168.1.1 NTP_ENABLED=no + +# 'etcd' cluster must have odd member(s) (1,3,5,...) +# variable 'NODE_NAME' is the distinct name of a member in 'etcd' cluster +[etcd] +192.168.1.1 NODE_NAME=etcd1 +192.168.1.2 NODE_NAME=etcd2 +192.168.1.3 NODE_NAME=etcd3 + +[kube-master] +192.168.1.2 +192.168.1.3 + +[kube-node] +192.168.1.1 +192.168.1.4 + +# set 'NEW_INSTALL': 'yes' to install a harbor server; 'no' to integrate with existed one +# if not using domian，set 'HARBOR_DOMAIN=""' +[harbor] +#192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no + +[all:vars] +# --------- Main Variables --------------- +# Cluster Deployment Mode: allinone, single-master, multi-master +DEPLOY_MODE=multi-master + +# use loadballance service by the Cloud Providor, no 'lb' nodes needed +# cloud loadballance service: listen on tcp 8443, with kube-masters(apiservers) as backend servers +MASTER_IP="192.168.1.10" +KUBE_APISERVER="https://{{ MASTER_IP }}:8443" + +# Network plugins supported: calico, flannel +CLUSTER_NETWORK="flannel" + +# K8S Service CIDR, not overlap with node(host) networking +SERVICE_CIDR="10.68.0.0/16" + +# Cluster CIDR (Pod CIDR), not overlap with node(host) networking +CLUSTER_CIDR="172.20.0.0/16" + +# NodePort Range +NODE_PORT_RANGE="20000-40000" + +# Kubernetes SVC IP (usually assigned with the first available IP of 'SERVICE_CIDR') +CLUSTER_KUBERNETES_SVC_IP="10.68.0.1" + +# Cluster DNS Server's IP (assigned with an available IP of 'SERVICE_CIDR') +CLUSTER_DNS_SVC_IP="10.68.0.2" + +# Cluster DNS Domain +CLUSTER_DNS_DOMAIN="cluster.local." + +# Basic auth for apiserver (a random password will be gennerated on cluster setup) +BASIC_AUTH_USER="admin" +BASIC_AUTH_PASS="test1234" + +# -------- Additional Variables -------------------- +# Binaries Directory +bin_dir="/opt/kube/bin" + +# CA and other components cert/key Directory +ca_dir="/etc/kubernetes/ssl" + +# Deploy Directory (kubeasz workspace), don't change the default value right now +base_dir="/etc/ansible" diff --git a/example/hosts.m-masters.example b/example/hosts.m-masters.example new file mode 100644 index 0000000..fde5b4f --- /dev/null +++ b/example/hosts.m-masters.example @@ -0,0 +1,78 @@ +# 集群部署节点：一般为运行ansible 脚本的节点 +# 变量 NTP_ENABLED (=yes/no) 设置集群是否安装 chrony 时间同步 +[deploy] +192.168.1.1 NTP_ENABLED=no + +# etcd集群请提供如下NODE_NAME，注意etcd集群必须是1,3,5,7...奇数个节点 +[etcd] +192.168.1.1 NODE_NAME=etcd1 +192.168.1.2 NODE_NAME=etcd2 +192.168.1.3 NODE_NAME=etcd3 + +[kube-master] +192.168.1.1 +192.168.1.2 + +[kube-node] +192.168.1.3 +192.168.1.4 + +# 参数 NEW_INSTALL：yes表示新建，no表示使用已有harbor服务器 +# 如果不使用域名，可以设置 HARBOR_DOMAIN="" +[harbor] +#192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no + +# 负载均衡(目前已支持多于2节点，一般2节点就够了) 安装 haproxy+keepalived +[lb] +192.168.1.1 LB_ROLE=backup +192.168.1.2 LB_ROLE=master + +#【可选】外部负载均衡，用于自有环境负载转发 NodePort 暴露的服务等 +[ex-lb] +#192.168.1.6 LB_ROLE=backup EX_VIP=192.168.1.250 +#192.168.1.7 LB_ROLE=master EX_VIP=192.168.1.250 + +[all:vars] +# ---------集群主要参数--------------- +#集群部署模式：allinone, single-master, multi-master +DEPLOY_MODE=multi-master + +# 集群 MASTER IP即 LB节点VIP地址，为区别与默认apiserver端口，设置VIP监听的服务端口8443 +# 公有云上请使用云负载均衡内网地址和监听端口 +MASTER_IP="192.168.1.10" +KUBE_APISERVER="https://{{ MASTER_IP }}:8443" + +# 集群网络插件，目前支持calico, flannel, kube-router, cilium +CLUSTER_NETWORK="flannel" + +# 服务网段 (Service CIDR），注意不要与内网已有网段冲突 +SERVICE_CIDR="10.68.0.0/16" + +# POD 网段 (Cluster CIDR），注意不要与内网已有网段冲突 +CLUSTER_CIDR="172.20.0.0/16" + +# 服务端口范围 (NodePort Range) +NODE_PORT_RANGE="20000-40000" + +# kubernetes 服务 IP (预分配，一般是 SERVICE_CIDR 中第一个IP) +CLUSTER_KUBERNETES_SVC_IP="10.68.0.1" + +# 集群 DNS 服务 IP (从 SERVICE_CIDR 中预分配) +CLUSTER_DNS_SVC_IP="10.68.0.2" + +# 集群 DNS 域名 +CLUSTER_DNS_DOMAIN="cluster.local." + +# 集群basic auth 使用的用户名和密码 (运行时会生成随机密码) +BASIC_AUTH_USER="admin" +BASIC_AUTH_PASS="test1234" + +# ---------附加参数-------------------- +#默认二进制文件目录 +bin_dir="/opt/kube/bin" + +#证书目录 +ca_dir="/etc/kubernetes/ssl" + +#部署目录，即 ansible 工作目录，建议不要修改 +base_dir="/etc/ansible" diff --git a/example/hosts.m-masters.example.en b/example/hosts.m-masters.example.en new file mode 100644 index 0000000..8aab239 --- /dev/null +++ b/example/hosts.m-masters.example.en @@ -0,0 +1,79 @@ +# 'deploy' node, which the ansible-playbooks usually run on +# variable 'NTP_ENABLED(=yes/no)' enables/disables the NTP server 'chrony' +[deploy] +192.168.1.1 NTP_ENABLED=no + +# 'etcd' cluster must have odd member(s) (1,3,5,...) +# variable 'NODE_NAME' is the distinct name of a member in 'etcd' cluster +[etcd] +192.168.1.1 NODE_NAME=etcd1 +192.168.1.2 NODE_NAME=etcd2 +192.168.1.3 NODE_NAME=etcd3 + +[kube-master] +192.168.1.1 +192.168.1.2 + +[kube-node] +192.168.1.3 +192.168.1.4 + +# set 'NEW_INSTALL': 'yes' to install a harbor server; 'no' to integrate with existed one +# if not using domian，set 'HARBOR_DOMAIN=""' +[harbor] +#192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no + +# 'loadbalance' node, with 'haproxy+keepalived' installed +[lb] +192.168.1.1 LB_ROLE=backup +192.168.1.2 LB_ROLE=master + +# [optional] loadbalance for services with type 'NodePort' +[ex-lb] +#192.168.1.6 LB_ROLE=backup EX_VIP=192.168.1.250 +#192.168.1.7 LB_ROLE=master EX_VIP=192.168.1.250 + +[all:vars] +# --------- Main Variables --------------- +# Cluster Deployment Mode: allinone, single-master, multi-master +DEPLOY_MODE=multi-master + +# Cluster's Master IP, generated by 'keepalived' daemon on a 'lb' node here +# 'haproxy' daemon listens on port 8443, directs requests to real apiservers on port 6443 +MASTER_IP="192.168.1.10" +KUBE_APISERVER="https://{{ MASTER_IP }}:8443" + +# Network plugins supported: calico, flannel, kube-router, cilium +CLUSTER_NETWORK="flannel" + +# K8S Service CIDR, not overlap with node(host) networking +SERVICE_CIDR="10.68.0.0/16" + +# Cluster CIDR (Pod CIDR), not overlap with node(host) networking +CLUSTER_CIDR="172.20.0.0/16" + +# NodePort Range +NODE_PORT_RANGE="20000-40000" + +# Kubernetes SVC IP (usually assigned with the first available IP of 'SERVICE_CIDR') +CLUSTER_KUBERNETES_SVC_IP="10.68.0.1" + +# Cluster DNS Server's IP (assigned with an available IP of 'SERVICE_CIDR') +CLUSTER_DNS_SVC_IP="10.68.0.2" + +# Cluster DNS Domain +CLUSTER_DNS_DOMAIN="cluster.local." + +# Basic auth for apiserver (a random password will be gennerated on cluster setup) +BASIC_AUTH_USER="admin" +BASIC_AUTH_PASS="test1234" + +# -------- Additional Variables -------------------- +# Binaries Directory +bin_dir="/opt/kube/bin" + +# CA and other components cert/key Directory +ca_dir="/etc/kubernetes/ssl" + +# Deploy Directory (kubeasz workspace), don't change the default value right now +base_dir="/etc/ansible" diff --git a/example/hosts.s-master.example b/example/hosts.s-master.example new file mode 100644 index 0000000..610a9b3 --- /dev/null +++ b/example/hosts.s-master.example @@ -0,0 +1,69 @@ +# 集群部署节点：一般为运行ansible 脚本的节点 +# 变量 NTP_ENABLED (=yes/no) 设置集群是否安装 chrony 时间同步 +[deploy] +192.168.1.1 NTP_ENABLED=no + +# etcd集群请提供如下NODE_NAME，请注意etcd集群必须是1,3,5,7...奇数个节点 +[etcd] +192.168.1.1 NODE_NAME=etcd1 + +[kube-master] +192.168.1.1 + +[kube-node] +192.168.1.2 +192.168.1.3 + +# 参数 NEW_INSTALL：yes表示新建，no表示使用已有harbor服务器 +# 如果不使用域名，可以设置 HARBOR_DOMAIN="" +[harbor] +#192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no + +#【可选】外部负载均衡，用于自有环境负载转发 NodePort 暴露的服务等 +[ex-lb] +#192.168.1.6 LB_ROLE=backup EX_VIP=192.168.1.250 +#192.168.1.7 LB_ROLE=master EX_VIP=192.168.1.250 + +[all:vars] +# ---------集群主要参数--------------- +#集群部署模式：allinone, single-master, multi-master +DEPLOY_MODE=single-master + +#集群 MASTER IP，自动生成 +MASTER_IP="{{ groups['kube-master'][0] }}" +KUBE_APISERVER="https://{{ MASTER_IP }}:6443" + +# 集群网络插件，目前支持calico, flannel, kube-router, cilium +CLUSTER_NETWORK="flannel" + +# 服务网段 (Service CIDR），注意不要与内网已有网段冲突 +SERVICE_CIDR="10.68.0.0/16" + +# POD 网段 (Cluster CIDR），注意不要与内网已有网段冲突 +CLUSTER_CIDR="172.20.0.0/16" + +# 服务端口范围 (NodePort Range) +NODE_PORT_RANGE="20000-40000" + +# kubernetes 服务 IP (预分配，一般是 SERVICE_CIDR 中第一个IP) +CLUSTER_KUBERNETES_SVC_IP="10.68.0.1" + +# 集群 DNS 服务 IP (从 SERVICE_CIDR 中预分配) +CLUSTER_DNS_SVC_IP="10.68.0.2" + +# 集群 DNS 域名 +CLUSTER_DNS_DOMAIN="cluster.local." + +# 集群basic auth 使用的用户名和密码 (运行时会生成随机密码) +BASIC_AUTH_USER="admin" +BASIC_AUTH_PASS="test1234" + +# ---------附加参数-------------------- +#默认二进制文件目录 +bin_dir="/opt/kube/bin" + +#证书目录 +ca_dir="/etc/kubernetes/ssl" + +#部署目录，即 ansible 工作目录 +base_dir="/etc/ansible" diff --git a/example/hosts.s-master.example.en b/example/hosts.s-master.example.en new file mode 100644 index 0000000..d03b9b1 --- /dev/null +++ b/example/hosts.s-master.example.en @@ -0,0 +1,70 @@ +# 'deploy' node, which the ansible-playbooks usually run on +# variable 'NTP_ENABLED(=yes/no)' enables/disables the NTP server 'chrony' +[deploy] +192.168.1.1 NTP_ENABLED=no + +# 'etcd' cluster must have odd member(s) (1,3,5,...) +# variable 'NODE_NAME' is the distinct name of a member in 'etcd' cluster +[etcd] +192.168.1.1 NODE_NAME=etcd1 + +[kube-master] +192.168.1.1 + +[kube-node] +192.168.1.2 +192.168.1.3 + +# set 'NEW_INSTALL': 'yes' to install a harbor server; 'no' to integrate with existed one +# if not using domian，set 'HARBOR_DOMAIN=""' +[harbor] +#192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no + +# [optional] loadbalance for services with type 'NodePort' +[ex-lb] +#192.168.1.6 LB_ROLE=backup EX_VIP=192.168.1.250 +#192.168.1.7 LB_ROLE=master EX_VIP=192.168.1.250 + +[all:vars] +# --------- Main Variables --------------- +# Cluster Deployment Mode: allinone, single-master, multi-master +DEPLOY_MODE=single-master + +# Cluster's Master IP, auto generated +MASTER_IP="{{ groups['kube-master'][0] }}" +KUBE_APISERVER="https://{{ MASTER_IP }}:6443" + +# Network plugins supported: calico, flannel, kube-router, cilium +CLUSTER_NETWORK="flannel" + +# K8S Service CIDR, not overlap with node(host) networking +SERVICE_CIDR="10.68.0.0/16" + +# Cluster CIDR (Pod CIDR), not overlap with node(host) networking +CLUSTER_CIDR="172.20.0.0/16" + +# NodePort Range +NODE_PORT_RANGE="20000-40000" + +# Kubernetes SVC IP (usually assigned with the first available IP of 'SERVICE_CIDR') +CLUSTER_KUBERNETES_SVC_IP="10.68.0.1" + +# Cluster DNS Server's IP (assigned with an available IP of 'SERVICE_CIDR') +CLUSTER_DNS_SVC_IP="10.68.0.2" + +# Cluster DNS Domain +CLUSTER_DNS_DOMAIN="cluster.local." + +# Basic auth for apiserver (a random password will be gennerated on cluster setup) +BASIC_AUTH_USER="admin" +BASIC_AUTH_PASS="test1234" + +# -------- Additional Variables -------------------- +# Binaries Directory +bin_dir="/opt/kube/bin" + +# CA and other components cert/key Directory +ca_dir="/etc/kubernetes/ssl" + +# Deploy Directory (kubeasz workspace), don't change the default value right now +base_dir="/etc/ansible" diff --git a/manifests/dashboard/1.6.3/kubernetes-dashboard.yaml b/manifests/dashboard/1.6.3/kubernetes-dashboard.yaml new file mode 100644 index 0000000..ba21596 --- /dev/null +++ b/manifests/dashboard/1.6.3/kubernetes-dashboard.yaml @@ -0,0 +1,100 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration to deploy release version of the Dashboard UI compatible with +# Kubernetes 1.6 (RBAC enabled). +# +# Example usage: kubectl create -f + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard + labels: + k8s-app: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system +--- +kind: Deployment +apiVersion: apps/v1beta1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + spec: + containers: + - name: kubernetes-dashboard + #image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.3 + image: mirrorgooglecontainers/kubernetes-dashboard-amd64:v1.6.3 + ports: + - containerPort: 9090 + protocol: TCP + args: + # Uncomment the following line to manually specify Kubernetes API server Host + # If not specified, Dashboard will attempt to auto discover the API server and connect + # to it. Uncomment only if the default does not work. + # - --apiserver-host=http://my-address:port + livenessProbe: + httpGet: + path: / + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 + serviceAccountName: kubernetes-dashboard + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule +--- +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + name: kubernetes-dashboard + namespace: kube-system +spec: + ports: + - port: 80 + targetPort: 9090 + selector: + k8s-app: kubernetes-dashboard + type: NodePort diff --git a/manifests/dashboard/admin-user-sa-rbac.yaml b/manifests/dashboard/admin-user-sa-rbac.yaml new file mode 100644 index 0000000..667de88 --- /dev/null +++ b/manifests/dashboard/admin-user-sa-rbac.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admin-user + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: admin-user +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: admin-user + namespace: kube-system + diff --git a/manifests/dashboard/kubernetes-dashboard.yaml b/manifests/dashboard/kubernetes-dashboard.yaml new file mode 100644 index 0000000..a8d8929 --- /dev/null +++ b/manifests/dashboard/kubernetes-dashboard.yaml @@ -0,0 +1,165 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------------------- Dashboard Secret ------------------- # + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-certs + namespace: kube-system +type: Opaque + +--- +# ------------------- Dashboard Service Account ------------------- # + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + +--- +# ------------------- Dashboard Role & Role Binding ------------------- # + +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubernetes-dashboard-minimal + namespace: kube-system +rules: + # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret. +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + # Allow Dashboard to create 'kubernetes-dashboard-settings' config map. +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + # Allow Dashboard to get metrics from heapster. +- apiGroups: [""] + resources: ["services"] + resourceNames: ["heapster"] + verbs: ["proxy"] +- apiGroups: [""] + resources: ["services/proxy"] + resourceNames: ["heapster", "http:heapster:", "https:heapster:"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubernetes-dashboard-minimal + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard-minimal +subjects: +- kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system + +--- +# ------------------- Dashboard Deployment ------------------- # + +kind: Deployment +apiVersion: apps/v1beta2 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + spec: + containers: + - name: kubernetes-dashboard + image: mirrorgooglecontainers/kubernetes-dashboard-amd64:v1.10.1 + ports: + - containerPort: 8443 + protocol: TCP + args: + - --auto-generate-certificates + # Uncomment the following line to manually specify Kubernetes API server Host + # If not specified, Dashboard will attempt to auto discover the API server and connect + # to it. Uncomment only if the default does not work. + # - --apiserver-host=http://my-address:port + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs + # Create on-disk volume to store exec logs + - mountPath: /tmp + name: tmp-volume + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + +--- +# ------------------- Dashboard Service ------------------- # + +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + name: kubernetes-dashboard + namespace: kube-system +spec: + ports: + - port: 443 + targetPort: 8443 + selector: + k8s-app: kubernetes-dashboard + type: NodePort diff --git a/manifests/dashboard/read-user-sa-rbac.yaml b/manifests/dashboard/read-user-sa-rbac.yaml new file mode 100644 index 0000000..3f832cd --- /dev/null +++ b/manifests/dashboard/read-user-sa-rbac.yaml @@ -0,0 +1,149 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dashboard-read-user + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: dashboard-read-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: dashboard-read-clusterrole +subjects: +- kind: ServiceAccount + name: dashboard-read-user + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dashboard-read-clusterrole +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - persistentvolumeclaims + - pods + - replicationcontrollers + - replicationcontrollers/scale + - serviceaccounts + - services + - nodes + - persistentvolumeclaims + - persistentvolumes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - bindings + - events + - limitranges + - namespaces/status + - pods/log + - pods/status + - replicationcontrollers/status + - resourcequotas + - resourcequotas/status + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - daemonsets + - deployments + - deployments/scale + - replicasets + - replicasets/scale + - statefulsets + verbs: + - get + - list + - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - daemonsets + - deployments + - deployments/scale + - ingresses + - networkpolicies + - replicasets + - replicasets/scale + - replicationcontrollers/scale + verbs: + - get + - list + - watch +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - volumeattachments + verbs: + - get + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - roles + - rolebindings + verbs: + - get + - list + - watch diff --git a/manifests/dashboard/ui-admin-rbac.yaml b/manifests/dashboard/ui-admin-rbac.yaml new file mode 100644 index 0000000..ee39a7d --- /dev/null +++ b/manifests/dashboard/ui-admin-rbac.yaml @@ -0,0 +1,27 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ui-admin +rules: +- apiGroups: + - "" + resources: + - services + - services/proxy + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ui-admin-binding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ui-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: admin diff --git a/manifests/dashboard/ui-read-rbac.yaml b/manifests/dashboard/ui-read-rbac.yaml new file mode 100644 index 0000000..b03c6b4 --- /dev/null +++ b/manifests/dashboard/ui-read-rbac.yaml @@ -0,0 +1,29 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ui-read +rules: +- apiGroups: + - "" + resources: + - services + - services/proxy + verbs: + - get + - list + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ui-read-binding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ui-read +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: readonly diff --git a/manifests/efk/es-dynamic-pv/es-statefulset.yaml b/manifests/efk/es-dynamic-pv/es-statefulset.yaml new file mode 100644 index 0000000..0f5b111 --- /dev/null +++ b/manifests/efk/es-dynamic-pv/es-statefulset.yaml @@ -0,0 +1,117 @@ +# RBAC authn and authz +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elasticsearch-logging + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: elasticsearch-logging + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "" + resources: + - "services" + - "namespaces" + - "endpoints" + verbs: + - "get" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: kube-system + name: elasticsearch-logging + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +subjects: +- kind: ServiceAccount + name: elasticsearch-logging + namespace: kube-system + apiGroup: "" +roleRef: + kind: ClusterRole + name: elasticsearch-logging + apiGroup: "" +--- +# Elasticsearch deployment itself +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: elasticsearch-logging + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + version: v5.6.4 + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + serviceName: elasticsearch-logging + replicas: 2 + selector: + matchLabels: + k8s-app: elasticsearch-logging + version: v5.6.4 + template: + metadata: + labels: + k8s-app: elasticsearch-logging + version: v5.6.4 + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: elasticsearch-logging + containers: + #- image: gcr.io/google-containers/elasticsearch:v5.6.4 + - image: mirrorgooglecontainers/elasticsearch:v5.6.4 + name: elasticsearch-logging + resources: + # need more cpu upon initialization, therefore burstable class + limits: + cpu: 1000m + requests: + cpu: 100m + ports: + - containerPort: 9200 + name: db + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + volumeMounts: + - name: elasticsearch-logging + mountPath: /data + env: + - name: "NAMESPACE" + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # Elasticsearch requires vm.max_map_count to be at least 262144. + # If your OS already sets up this number to a higher value, feel free + # to remove this init container. + initContainers: + - image: alpine:3.6 + command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"] + name: elasticsearch-logging-init + securityContext: + privileged: true + volumeClaimTemplates: + - metadata: + name: elasticsearch-logging + spec: + accessModes: [ "ReadWriteMany" ] + storageClassName: "nfs-dynamic-class" + resources: + requests: + storage: 4Gi diff --git a/manifests/efk/es-service.yaml b/manifests/efk/es-service.yaml new file mode 100644 index 0000000..3c45e5c --- /dev/null +++ b/manifests/efk/es-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch-logging + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "Elasticsearch" +spec: + ports: + - port: 9200 + protocol: TCP + targetPort: db + clusterIP: None + selector: + k8s-app: elasticsearch-logging diff --git a/manifests/efk/es-static-pv/es-pv0.yaml b/manifests/efk/es-static-pv/es-pv0.yaml new file mode 100644 index 0000000..7686cc9 --- /dev/null +++ b/manifests/efk/es-static-pv/es-pv0.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-es-0 +spec: + capacity: + storage: 4Gi + accessModes: + - ReadWriteMany + volumeMode: Filesystem + persistentVolumeReclaimPolicy: Recycle + storageClassName: "es-storage-class" + nfs: + # 根据实际共享目录修改 + path: /share/es0 + # 根据实际 nfs服务器地址修改 + server: 192.168.1.208 diff --git a/manifests/efk/es-static-pv/es-pv1.yaml b/manifests/efk/es-static-pv/es-pv1.yaml new file mode 100644 index 0000000..b41309e --- /dev/null +++ b/manifests/efk/es-static-pv/es-pv1.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-es-1 +spec: + capacity: + storage: 4Gi + accessModes: + - ReadWriteMany + volumeMode: Filesystem + persistentVolumeReclaimPolicy: Recycle + storageClassName: "es-storage-class" + nfs: + # 根据实际共享目录修改 + path: /share/es1 + # 根据实际 nfs服务器地址修改 + server: 192.168.1.208 diff --git a/manifests/efk/es-static-pv/es-pv2.yaml b/manifests/efk/es-static-pv/es-pv2.yaml new file mode 100644 index 0000000..f1f4c05 --- /dev/null +++ b/manifests/efk/es-static-pv/es-pv2.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-es-2 +spec: + capacity: + storage: 4Gi + accessModes: + - ReadWriteMany + volumeMode: Filesystem + persistentVolumeReclaimPolicy: Recycle + storageClassName: "es-storage-class" + nfs: + # 根据实际共享目录修改 + path: /share/es2 + # 根据实际 nfs服务器地址修改 + server: 192.168.1.208 diff --git a/manifests/efk/es-static-pv/es-statefulset.yaml b/manifests/efk/es-static-pv/es-statefulset.yaml new file mode 100644 index 0000000..f8e7f4f --- /dev/null +++ b/manifests/efk/es-static-pv/es-statefulset.yaml @@ -0,0 +1,117 @@ +# RBAC authn and authz +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elasticsearch-logging + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: elasticsearch-logging + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "" + resources: + - "services" + - "namespaces" + - "endpoints" + verbs: + - "get" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: kube-system + name: elasticsearch-logging + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +subjects: +- kind: ServiceAccount + name: elasticsearch-logging + namespace: kube-system + apiGroup: "" +roleRef: + kind: ClusterRole + name: elasticsearch-logging + apiGroup: "" +--- +# Elasticsearch deployment itself +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: elasticsearch-logging + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + version: v5.6.4 + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + serviceName: elasticsearch-logging + replicas: 2 + selector: + matchLabels: + k8s-app: elasticsearch-logging + version: v5.6.4 + template: + metadata: + labels: + k8s-app: elasticsearch-logging + version: v5.6.4 + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: elasticsearch-logging + containers: + #- image: gcr.io/google-containers/elasticsearch:v5.6.4 + - image: mirrorgooglecontainers/elasticsearch:v5.6.4 + name: elasticsearch-logging + resources: + # need more cpu upon initialization, therefore burstable class + limits: + cpu: 1000m + requests: + cpu: 100m + ports: + - containerPort: 9200 + name: db + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + volumeMounts: + - name: elasticsearch-logging + mountPath: /data + env: + - name: "NAMESPACE" + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # Elasticsearch requires vm.max_map_count to be at least 262144. + # If your OS already sets up this number to a higher value, feel free + # to remove this init container. + initContainers: + - image: alpine:3.6 + command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"] + name: elasticsearch-logging-init + securityContext: + privileged: true + volumeClaimTemplates: + - metadata: + name: elasticsearch-logging + spec: + accessModes: [ "ReadWriteMany" ] + storageClassName: "es-storage-class" + resources: + requests: + storage: 4Gi diff --git a/manifests/efk/es-without-pv/es-statefulset.yaml b/manifests/efk/es-without-pv/es-statefulset.yaml new file mode 100644 index 0000000..4bd6584 --- /dev/null +++ b/manifests/efk/es-without-pv/es-statefulset.yaml @@ -0,0 +1,111 @@ +# RBAC authn and authz +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elasticsearch-logging + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: elasticsearch-logging + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "" + resources: + - "services" + - "namespaces" + - "endpoints" + verbs: + - "get" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: kube-system + name: elasticsearch-logging + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +subjects: +- kind: ServiceAccount + name: elasticsearch-logging + namespace: kube-system + apiGroup: "" +roleRef: + kind: ClusterRole + name: elasticsearch-logging + apiGroup: "" +--- +# Elasticsearch deployment itself +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: elasticsearch-logging + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + version: v5.6.4 + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + serviceName: elasticsearch-logging + replicas: 2 + selector: + matchLabels: + k8s-app: elasticsearch-logging + version: v5.6.4 + template: + metadata: + labels: + k8s-app: elasticsearch-logging + version: v5.6.4 + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: elasticsearch-logging + containers: + #- image: gcr.io/google-containers/elasticsearch:v5.6.4 + - image: mirrorgooglecontainers/elasticsearch:v5.6.4 + name: elasticsearch-logging + resources: + # need more cpu upon initialization, therefore burstable class + limits: + cpu: 1000m + requests: + cpu: 100m + ports: + - containerPort: 9200 + name: db + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + volumeMounts: + - name: elasticsearch-logging + mountPath: /data + env: + - name: "NAMESPACE" + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: elasticsearch-logging + emptyDir: {} + # Elasticsearch requires vm.max_map_count to be at least 262144. + # If your OS already sets up this number to a higher value, feel free + # to remove this init container. + initContainers: + - image: alpine:3.6 + command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"] + name: elasticsearch-logging-init + securityContext: + privileged: true diff --git a/manifests/efk/fluentd-es-configmap.yaml b/manifests/efk/fluentd-es-configmap.yaml new file mode 100644 index 0000000..09fbad0 --- /dev/null +++ b/manifests/efk/fluentd-es-configmap.yaml @@ -0,0 +1,373 @@ +kind: ConfigMap +apiVersion: v1 +data: + containers.input.conf: |- + # This configuration file for Fluentd / td-agent is used + # to watch changes to Docker log files. The kubelet creates symlinks that + # capture the pod name, namespace, container name & Docker container ID + # to the docker logs for pods in the /var/log/containers directory on the host. + # If running this fluentd configuration in a Docker container, the /var/log + # directory should be mounted in the container. + # + # These logs are then submitted to Elasticsearch which assumes the + # installation of the fluent-plugin-elasticsearch & the + # fluent-plugin-kubernetes_metadata_filter plugins. + # See https://github.com/uken/fluent-plugin-elasticsearch & + # https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter for + # more information about the plugins. + # + # Example + # ======= + # A line in the Docker log file might look like this JSON: + # + # {"log":"2014/09/25 21:15:03 Got request with path wombat\n", + # "stream":"stderr", + # "time":"2014-09-25T21:15:03.499185026Z"} + # + # The time_format specification below makes sure we properly + # parse the time format produced by Docker. This will be + # submitted to Elasticsearch and should appear like: + # $ curl 'http://elasticsearch-logging:9200/_search?pretty' + # ... + # { + # "_index" : "logstash-2014.09.25", + # "_type" : "fluentd", + # "_id" : "VBrbor2QTuGpsQyTCdfzqA", + # "_score" : 1.0, + # "_source":{"log":"2014/09/25 22:45:50 Got request with path wombat\n", + # "stream":"stderr","tag":"docker.container.all", + # "@timestamp":"2014-09-25T22:45:50+00:00"} + # }, + # ... + # + # The Kubernetes fluentd plugin is used to write the Kubernetes metadata to the log + # record & add labels to the log record if properly configured. This enables users + # to filter & search logs on any metadata. + # For example a Docker container's logs might be in the directory: + # + # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b + # + # and in the file: + # + # 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log + # + # where 997599971ee6... is the Docker ID of the running container. + # The Kubernetes kubelet makes a symbolic link to this file on the host machine + # in the /var/log/containers directory which includes the pod name and the Kubernetes + # container name: + # + # synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log + # -> + # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log + # + # The /var/log directory on the host is mapped to the /var/log directory in the container + # running this instance of Fluentd and we end up collecting the file: + # + # /var/log/containers/synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log + # + # This results in the tag: + # + # var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log + # + # The Kubernetes fluentd plugin is used to extract the namespace, pod name & container name + # which are added to the log message as a kubernetes field object & the Docker container ID + # is also added under the docker field object. + # The final tag is: + # + # kubernetes.var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log + # + # And the final log record look like: + # + # { + # "log":"2014/09/25 21:15:03 Got request with path wombat\n", + # "stream":"stderr", + # "time":"2014-09-25T21:15:03.499185026Z", + # "kubernetes": { + # "namespace": "default", + # "pod_name": "synthetic-logger-0.25lps-pod", + # "container_name": "synth-lgr" + # }, + # "docker": { + # "container_id": "997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b" + # } + # } + # + # This makes it easier for users to search for logs by pod name or by + # the name of the Kubernetes container regardless of how many times the + # Kubernetes pod has been restarted (resulting in a several Docker container IDs). + + # Json Log Example: + # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} + # CRI Log Example: + # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here + + type tail + path /var/log/containers/*.log + pos_file /var/log/es-containers.log.pos + time_format %Y-%m-%dT%H:%M:%S.%NZ + tag kubernetes.* + read_from_head true + format multi_format + + format json + time_key time + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + format /^(? + + system.input.conf: |- + # Example: + # 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081 + + type tail + format /^(?