Shell 实战项目
1. 前言
通过之前的学习，我们已经分章节系统性的讲解了 Shell 脚本的知识，本章节我们通过两个项目实战，进一步详解 Shell 在实战中的知识组合及灵活运用。
2. 系统巡检
需要对 Linux 服务器，巡检系统 / 服务 / 端口各指标及参数，编写系统检查脚本，系统信息到脚本运行的 logs 目录下。
通常系统的状态都在 /proc/ 目录下，对中的关注系统指标数字进行过滤处理，得到系统状态，其中涉及数组的需要计算，可以利用，对每需要巡检的结果利用单独编写，最后利用 main 其他得出最终结果。
核心利用 sed ：
#!/bin/bash# auth:kaliarch# func:sys info check# version:v1.0# sys:centos6.x/7.x# 判断是否为root，如果非root则需要root执行脚本[ $(id -u) -gt ] && echo "请用root执行此脚本！" && exit sysversion=$(rpm -q centos-release|cut -- -)line="-------------------------------------------------"# 创建日志目录[ - logs ] || mkdir logs# 定义日志sys_check_file="logs/$(ip a show dev eth0|grep -w inet|awk '{print $2}'|awk -F '/' '{print $1}')-`date +%Y%m%d`.txt"# 系统信息function get__info() {
Physical_s=$(grep "physical id" /proc/info| sort | uniq | wc -l)Virt_s=$(grep "processor" /proc/info | wc -l)_Kernels=$(grep "cores" /proc/info|uniq| awk -F ': ' '{print $2}')_Type=$(grep "model name" /proc/info | awk -F ': ' '{print $2}' | sort | uniq)_Arch=$(uname -m)cat <<EOF | column -t
信息:
物理个数: $Physical_s
逻辑个数: $Virt_s
每核心数: $_Kernels
型号: $_Type
架构: $_Arch
EOF
}# 系统function get_mem_info() {
check_mem=$(free -m)MemTotal=$(grep MemTotal /proc/meminfo| awk '{print $2}') #KBMemFree=$(grep MemFree /proc/meminfo| awk '{print $2}') #KBlet MemUsed=MemTotal-MemFree
MemPercent=$(awk "BEGIN {if($MemTotal==0){printf 100}else{printf \"%.2f\",$MemUsed*100/$MemTotal}}")report_MemTotal="$((MemTotal/1024))""MB" #内存总容量(MB)report_MemFree="$((MemFree/1024))""MB" #内存剩余(MB)report_MemUsedPercent="$(awk "BEGIN {if($MemTotal==){printf }else{printf \"%.\",$MemUsed*/$MemTotal}}")""%" #内存使用率%cat <<EOF
：
${check_mem}
EOF
}# 系统网络信息function get_net_info() {
pri_ipadd=$(ip show dev eth0|grep -w inet|awk '{print $2}'|awk -F '/' '{print $1}')pub_ipadd=$(curl ifcon.me -s)gateway=$(ip route | grep default | awk '{print $3}')mac_info=$(ip link| egrep -v "lo"|grep link|awk '{print $2}')dns_con=$(egrep -v "^$|^#" /etc/resolv.conf)route_info=$(route -n)cat <<EOF | column -t
IP信息:
系统公网地址: ${pub_ipadd}
系统私网地址: ${pri_ipadd}
网关地址: ${gateway}
MAC地址: ${mac_info}
路由信息:
${route_info}
DNS 信息:
${dns_con}
EOF
}# 系统磁盘信息function get_disk_info() {
disk_info=$(fdisk -l|grep "Disk /dev"|cut -, -)disk_use=$( -hTP|awk '$2!="tmpfs"{print}')disk_inode=$( -hiP|awk '$1!="tmpfs"{print}')cat <<EOF
磁盘信息:
${disk_info}
磁盘使用:
${disk_use}
inode信息:
${disk_inode}
EOF
}# 系统信息function get_systatus_info() {
sys_os=$(uname -o)sys_release=$(cat /etc/redhat-release)sys_kernel=$(uname -r)sys_hostname=$(hostname)sys_selinux=$(getenforce)sys_lang=$(echo $LANG)sys_lastreboot=$(who - | awk '{print $3,$4}')sys_runtime=$(uptime |awk '{print $3,$4}'|cut -, -)sys_time=$(date)sys_load=$(uptime |cut -: -)cat <<EOF | column -t
系统信息：
系统: ${sys_os}
发行版本: ${sys_release}
系统内核: ${sys_kernel}
主机名: ${sys_hostname}
selinux状态: ${sys_selinux}
系统语言: ${sys_lang}
系统当前时间: ${sys_time}
系统最后重启时间: ${sys_lastreboot}
系统运行时间: ${sys_runtime}
系统: ${sys_load}
EOF
}# 服务信息function get_service_info() {
port_listen=$(netstat -lntup|grep -v "Active Internet")kernel_con=$(sysctl -p >/dev/null)if [ ${sysversion} -gt ];thenservice_con=$(syctl list-unit-files --type=service --state=enabled|grep "enabled")run_service=$(syctl list-units --type=service --state=running |grep ".service")elseservice_con=$(/sbin/chkcon | grep -E ":on|:启用" |column -t)run_service=$(/sbin/service --status-all|grep -E "running")fi
cat <<EOF
服务启动配置：
${service_con}
${line}
运行的服务：
${run_service}
${line}
监听端口：
${port_listen}
${line}
内核参考配置:
${kernel_con}
}# 系统信息function get_sys_user() {
login_user=$(awk -F: '{if ($NF=="/bin/bash") print $0}' /etc/passwd)ssh_con=$(egrep -v "^#|^$" /etc/ssh/sshd_con)sudo_con=$(egrep -v "^#|^$" /etc/sudoers |grep -v "^Defaults")host_con=$(egrep -v "^#|^$" /etc/hosts)crond_con=$(for cronuser in /var/spool/cron/* ;do ls ${cronuser} >/dev/null|cut -/ -;egrep -v "^$|^#" ${cronuser} >/dev/null;echo "";done)cat <<EOF
系统:
${login_user}
${line}
ssh 配置信息:
${ssh_con}
${line}
sudo 配置:
${sudo_con}
${line}
定时任务配置:
${crond_con}
${line}
hosts 信息:
${host_con}
EOF
}# 进程信息function process_top_info() {
top_title=$(top - n1|head -|tail -)_top10=$(top -n1 | head - | tail -)mem_top10=$(top - n1|head -|tail -|sort -k10 -r)cat <<EOF
占用top10:
${top_title}
${_top10}
内存占用top10:
${top_title}
${mem_top10}
EOF
}# 信息汇总function sys_check() {
get__info
echo ${line}
get_mem_info
echo ${line}
get_net_info
echo ${line}
get_disk_info
echo ${line}
get_systatus_info
echo ${line}
get_service_info
echo ${line}
get_sys_user
echo ${line}
process_top_info
}
sys_check > ${sys_check_file}
[root@master workspace]# bash sys_check.sh [root@master workspace]# lslogs sys_check.sh[root@master workspace]# cat logs/172.16.60.2-20200.txt 信息: 物理个数: 逻辑个数: 每核心数: 型号: QEMU Virtual 架构: x86_64-------------------------------------------------： total used free shared buff/cache available Mem: Swap: -------------------------------------------------IP信息: 系统公网地址: 4x.xxx.xxx 系统私网地址: .网关地址: .MAC地址: ::::: 路由信息: Kernel IP routing tableDestination Gateway Genmask Flags Metric Ref Use Iface. . . UG eth0. . . UG tunl0. . . U eth0. . . U docker0 DNS 信息: nameserver .nameserver .-------------------------------------------------磁盘信息:Disk /dev/vda: GBDisk /dev/vdb: MB 磁盘使用: Filesy Type Size Used Avail Use% Mounted on/dev/vda1 ext4 20G .0G 14G % /devtmpfs devtmpfs .9G .9G % /dev/dev/vdc ext4 197G 46G 142G % /data/dev/vdd xfs 500G 53G 448G % /var/local/osd0 inode信息: Filesy Inodes IUsed IFree IUse% Mounted on/dev/vda1 .3M 54K .2M % /devtmpfs .0M .0M % /dev/dev/vdc 13M .3M 12M % /data/dev/vdd 250M 14K 250M % /var/local/osd0 overlay 13M .3M 12M % /merged 系统: GNU/Linux 发行版本: CentOS Linux release . (Core)系统内核: .-.el7.x86_64 主机名: master selinux状态: Permissive 系统语言: en_US.UTF 系统当前时间: Sun Apr :: CST 系统最后重启时间: -- : 系统运行时间: days 系统: , , -------------------------------------------------服务启动配置: auditd.service enabled autovt@.service enabled ceph-mon@.service enabled etcd.service enabled gapd.service enabled getty@.service enabled irqbalance.service enabled kdump.service enabled kubelet.service enabled microcode.service enabled NetworkManager-dispatcher.service enabled NetworkManager.service enabled postfix.service enabled rpcbind.service enabled rsyslog.service enabled smarteye-server-agent.service enabled sshd.service enabled syd-readahead-collect.service enabled syd-readahead-drop.service enabled syd-readahead-replay.service enabled tuned.service enabled-------------------------------------------------运行的服务: auditd.service loaded active running S Auditing Service ceph-mon@master.service loaded active running Ceph cluster monitor daemon ceph-osd@0.service loaded active running Ceph object storage daemon chronyd.service loaded active running NTP client/server crond.service loaded active running Command Scheduler dbus.service loaded active running D-Bus Sy Message Bus docker.service loaded active running Docker Application Container Engineetcd.service loaded active running etcd docker wrapper gapd.service loaded active running guest agent for pitrix getty@tty1.service loaded active running Getty on tty1 gssproxy.service loaded active running GSSAPI Proxy Daemon irqbalance.service loaded active running irqbalance daemon kubelet.service loaded active running Kubernetes Kubelet Server NetworkManager.service loaded active running Network Manager polkit.service loaded active running Authorization Manager postfix.service loaded active running Postfix Mail Transport Agent rpcbind.service loaded active running RPC bind service rsyslog.service loaded active running Sy Logging Service smarteye-server-agent.service loaded active running The Smarteye Monitoring of server sshd.service loaded active running OpenSSH server daemon syd-journald.service loaded active running Journal Service syd-logind.service loaded active running Login Service syd-udevd.service loaded active running udev Kernel Device Manager tuned.service loaded active running Dynamic Sy Tuning Daemon-------------------------------------------------监听端口: Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp .: .:* LISTEN /calico-node tcp .: .:* LISTEN /etcd tcp .: .:* LISTEN /etcd tcp .: .:* LISTEN /./kube-rbac-pr tcp .: .:* LISTEN /node_exporter tcp .: .:* LISTEN /etcd tcp .: .:* LISTEN /kubelet tcp .: .:* LISTEN /syd tcp .: .:* LISTEN /ceph-osd tcp .: .:* LISTEN /ceph-osd tcp .: .:* LISTEN /ceph-osd tcp .: .:* LISTEN /bird tcp .: .:* LISTEN /ceph-osd tcp .: .:* LISTEN /node-cache tcp .: .:* LISTEN /sshd tcp .: .:* LISTEN /master tcp .: .:* LISTEN /kubelet tcp .: .:* LISTEN /ceph-mon tcp .: .:* LISTEN /node-cache tcp .: .:* LISTEN /kubelet tcp .: .:* LISTEN /kube-proxy tcp .: .:* LISTEN /kubelet tcp6 ::: :::* LISTEN /kube-schedule tcp6 ::: :::* LISTEN /kube-apiserver tcp6 ::: :::* LISTEN /kube-controll udp6 ::: :::* /syd udp6 ::: :::* /chronyd udp6 ::: :::* /rpcbind -------------------------------------------------内核参考配置: net.ipv4.ip_forward = net.bridge.bridge-nf-call-ip6tables = net.bridge.bridge-nf-call-iptables = net.ipv4.ip_local_reserved_ports = -net.bridge.bridge-nf-call-arptables = -------------------------------------------------系统: root:x:::root:/root:/bin/bash-------------------------------------------------ssh 配置信息: HostKey /etc/ssh/ssh_host_rsa_key HostKey /etc/ssh/ssh_host_ecdsa_key HostKey /etc/ssh/ssh_host_ed25519_key SyslogFacility AUTHPRIV AuthorizedKeysFile .ssh/authorized_keys PasswordAuthentication yes ChallengeResponseAuthentication noGSSAPIAuthentication noGSSAPICleanupCredentials noUsePAM yes X11Forwarding yes AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE AcceptEnv XMODIFIERS Subsy sftp /usr/libexec/openssh/sftp-server PermitRootLogin yes PermitEmptyPasswords noUseDNS no-------------------------------------------------sudo 配置: root ALL=(ALL) ALL%wheel ALL=(ALL) ALL-------------------------------------------------定时任务配置: root@weekly /usr/bin/docker sy prune -*/ * * * * /opt/etcd_back/etcd_backup.sh-------------------------------------------------hosts 信息:. localhost localhost.localdomain localhost4 localhost4.localdomain4 :: localhost6 localhost6.localdomain6 localhost6.localdomain. master. node01. node02. node03. master.cluster.local master. node01.cluster.local node01. node02.cluster.local node02. node03.cluster.local node03. harbor.devops.kubesphere.local. gitlab.devops.kubesphere.local-------------------------------------------------占用top10: PID USER PR NI VIRT RES SHR S % %MEM TIME+ COMMAND root S : kubelet root S : S : jaeger-operator root S : kube-controller root S : mixs root R : top root .074g S : etcd root S : syd root S : kthreadd root S : ksoftirqd/内存占用top10: PID USER PR NI VIRT RES SHR S % %MEM TIME+ COMMAND root S : kube-apiserver root .914g .834g S : java polkitd S : java root .188g S : etcd root S : kube-controller ceph S : ceph-mon root S : envoy root S : syd root R : top polkitd S : postgres
3 ELK 安装
对于软件安装，通常人工操作不仅繁琐耗时，且容易出错，可以用编写 Shell 来完成，后期如果有重复需求，可以一劳永逸，安装不易出错且提升效率，ELK 为开源日志系统 由 Elasticsearch、Logstash 和 Kibana 三部分组件组成。
可以针对需要按照的各种服务单独编写脚本来进行按照，其中利用了系统内的很多命令，也包含的操作等，在此刚好回顾我们之前学习的。
#!/bin/bash#mail:xuel@anchnet.com#data:2020/4/10#AutoInstall ELK scripts#Software:.4.1/logstash-5.4.1/filebeat-5.4.1/kibana-5.4.1clearecho "##########################################"echo "# Auto Install ELK. ##"echo "# Press Ctrl + C to cancel ##"echo "# Any key to continue ##"echo "##########################################"# 读入选择read -p
# 定义环境及目录变量software_dir="/usr/local/software"elasticsearch_url="https://artifacts.elastic.co/downloads/elasticsearch/.4.1.tar.gz"kibana_url="https://artifacts.elastic.co/downloads/kibana/kibana-5.4.1-linux-x86_64.tar.gz"logstash_url="https://artifacts.elastic.co/downloads/logstash/logstash-5.4.1.tar.gz"filebeat_url="https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.4.1-linux-x86_64.tar.gz"sys_version=`cat /etc/redhat-release |awk '{print $4}'|cut -d. -f1`IP=`ip addr|grep "inet "|grep -v 127.0.0.1|awk '{print $2}'|cut -d/ -f1`
jvm_conf="/usr/local/elasticsearch/con/jvm.options"sys_mem=`free -m|grep Mem:|awk '{print $2}'|awk '{sum+=$1} END {print sum/1024}'|cut -d. -f1`#下载软件，wget_fun() {if [ ! -d ${software_dir} ];thenmkdir -p ${software_dir} && cd ${software_dir}elsecd ${software_dir}fifor software in $elasticsearch_url $kibana_url $logstash_url $filebeat_urldowget -c $softwaredoneclear}# 初始化系统，安装java环境，设置主机，禁用防火墙init_sys() {[ -f /etc/init.d/functions ] && . /etc/init.d/functions[ "${sys_version}" != "7" ] && echo "Error:This Scripts Support Centos7.xx" && exit 1[ $(id -u) != "0" ] && echo "Error: You must be root to run this script" && exit 1sed -i "s/SELINUX=enforcing/SELINUX=disabled/" /etc/selinux/con
setenforce 0
yum install -y java-1.8.0-openjdk wget net-tools
hostnamectl set-hostname elk-server
syctl stop firewalld# /进程打开数cat >>/etc/s/limits.conf<<EOF
* soft nofile 65536
* hard nofile 65536
* soft nproc 65536
* hard nproc 65536
EOF}# 安装elasticsearchinstall_elasticsearch() {cd $software_dirtar zxf .4.1.tar.gzmv .4.1 /usr/local/elasticsearchmkdir -p /usr/local/elasticsearch/data /usr/local/elasticsearch/logs# useradd elasticsearchchown -R elasticsearch:elasticsearch /usr/local/elasticsearchecho "vm.max_map_count = 655360" >>/etc/sysctl.conf && sysctl -p# if [ ${sys_mem} -eq 0 ];thensed -i "s#`grep "^-Xmx" ${jvm_conf}`#"-Xmx512m"#g" ${jvm_conf}sed -i "s#`grep "^-Xms" ${jvm_conf}`#"-Xms512m"#g" ${jvm_conf}elsesed -i "s#`grep "^-Xmx" ${jvm_conf}`#"-Xmx${sys_mem}g"#g" ${jvm_conf}sed -i "s#`grep "^-Xms" ${jvm_conf}`#"-Xms${sys_mem}g"#g" ${jvm_conf}ficat >>/usr/local/elasticsearch/con/elasticsearch.yml<<EOF
cluster.name: my-application
node.name: elk-server
path.data: /usr/local/elasticsearch/data
path.logs: /usr/local/elasticsearch/logs
network.host: 127.0.0.1
http.port: 9200
discovery.zen.ping.unicast.hosts: ["elk-server"]
EOF# 启动es服务su - elasticsearch -c "nohup /usr/local/elasticsearch/bin/elasticsearch &"}#安装 logstashinstall_logstash() {cd $software_dirtar -zxf logstash-5.4.1.tar.gzmv logstash-5.4.1 /usr/local/logstash# cat>/usr/local/logstash/con/01-syslog.conf<<EOF
input {
beats {
port => "5044"
}
}
output {
elasticsearch {
hosts => "127.0.0.1:9200"
}
stdout { codec => rubydebug }
}
EOFnohup /usr/local/logstash/bin/logstash -f /usr/local/logstash/con/01-syslog.conf & >/dev/null}#安装 filebeatinstall_filebeat() {cd $software_dirtar -zxf filebeat-5.4.1-linux-x86_64.tar.gzmv filebeat-5.4.1-linux-x86_64 /usr/local/filebeatcat >/usr/local/filebeat/filebeat.yml<<EOF
filebeat.prospectors:
- input_type: log
paths:
- /var/log/*.log
output.logstash:
hosts: ["127.0.0.1:5044"]
EOFcd /usr/local/filebeat/nohup /usr/local/filebeat/filebeat & >/dev/null}#安装 kibanainstall_kibana() {cd $software_dirtar -zxf kibana-5.4.1-linux-x86_64.tar.gzmv kibana-5.4.1-linux-x86_64 /usr/local/kibana# cat >> /usr/local/kibana/con/kibana.yml <<EOF
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.url: "http://127.0.0.1:9200"
EOFnohup /usr/local/kibana/bin/kibana & >/dev/null}# 检测服务check() {port=$1program=$2check_port=`netstat -lntup|grep ${port}|wc -l`check_program=`ps -ef|grep ${program}|grep -v grep|wc -l`if [ $check_port -gt 0 ] && [ $check_program -gt 0 ];thenaction "${program} run is ok!" /bin/trueelseaction "${program} run is error!" /bin/falsefi}# 主统一main() {init_sys
wget_fun
install_elasticsearch
install_filebeat
install_logstash
install_kibanaecho -e "\033[32m Checking Elasticsearch...\033[0m"sleep 20
check :9200 "elasticsearch"echo -e "\033[32m Checking Logstash...\033[0m"sleep 2
check ":9600" "logstash"echo -e "\033[32m Checking Kibana...\033[0m"sleep 2
check ":5601" "kibana"action "ELK install is success!" /bin/trueecho "url:http://$IP:5601"}# 执行主main
@H__44@
安装完成访问 http://IP:5601 即可，注意 IP 地址为按照 ELK 的服务器 IP 地址。
4. 小结
本章节我们简单的列举了两个具体实例，来实战 Shell 脚本编写处理具体问题，至此就结束了 Shell 的全部章节，在此希望本文可以带给大家 Shell 学习的一些思路和。
学习 Shell 需要多动手实践，在日常编写中举一反三，针对脚本如何能无状态，更健壮，更灵活易维护，需要多次的，反复的执行验证，针对不同的应用场景，将数据抽象为参数进行传递，可以达到是事半功倍的。注意将有限的精力放在热点知识上，其他工具或命令的常用选项或参数牢记即可，其他不常用选项不建议死记硬背，活用 —help 或 man 手册查看，这样在后期的 Shell 编写中能够更加的高效。
602392714
清零编程群