本文共 17749 字，大约阅读时间需要 59 分钟。
只需要在服务器上创建两个文件即可：
1、vim config.txtjavapath=/usr/local/JDK/jdk1.7.0_75/bin/:/usr/java5_64/bin:.
export PATH=$PATH:/usr/local/JDK/jdk1.7.0_75/bin/:/usr/java5_64/bin:.export LANG="zh_CN.utf8"
export cronrebAdminlog="/home/xlcpt/wls_mon/xlcpt_domain/log/xlcpt_cronrebAdmin.log"
export cronrebManagedlog="/home/xlcpt/wls_mon/xlcpt_domain/log/xlcpt_cronrebManaged.log"export monAdminlog="/home/xlcpt/wls_mon/xlcpt_domain/log/xlcpt_monAdmin.log"export monManagedlog="/home/xlcpt/wls_mon/xlcpt_domain/log/xlcpt_monManaged.log"export DOMAINNAME="xlcpt_domain"
export DOMAIN_DIR="/home/xlcpt/domains/xlcpt_domain"export wls_jar="/home/weblogic/weblogic12/wlserver/server/lib/weblogic.jar"export LocalListenAddress="10.133.212.29"export AdminServerIP="10.133.212.29"export IsExistAdminServer="true"export userconfigfile="/home/xlcpt/wls_mon/xlcpt_domain/config/xlcpt-WebLogicConfig.properties"
export userkeyfile="/home/xlcpt/wls_mon/xlcpt_domain/config/xlcpt-WebLogicKey.properties"export start_interval=15
export start_count=24ManagedServer8002 10.133.212.29 8002
ManagedServer8003 10.133.212.29 8003AdminServer8001 10.133.212.29 8001
2、vim wls_mon.sh
usage(){ cat <<EOFUSAGE: wls_mon.sh [ -h ] [-c [ Admin|Managed ] ] [-a [ Admin|Managed ] ]OPTIONS:-h help,使用过程中遇到任何问题,请联系脚本提供者;-c [ Admin|Managed ] 定时重启weblogic主管或者受管服务;-a [ Admin|Managed ] 定时监控weblogic主管或者受管服务状态;EOFexit 0;}ftpserver()
{ ftpasswd=openssl des3 -d -k 123 -base64 -in /home/$(whoami)/.ftp41.propertiesftpfile="$1" #要发送的文件ftp -n <<! 2>/dev/null 1>&2open ${ftpip}user ${ftpuser} ${ftpasswd}cd ${ftpdir}binpromptput ${ftpfile}bye!} sendNotice()
{ #传过来的参数分别是:服务名称 监听地址 监听端口 [宕了|启动失败] [宕了但进程还存在][ $# -lt 4 ] && { echo "传递给sendNotice函数的参数不对,请检查!;exit"}tf="`hostname`_`date +'%y%m%d_%H%M%S'`.txt" #这里要加上秒,因为若同一个分钟内则生成的文件会覆盖掉[ "$4" = "down" ] && {cat < ${tf}`date +'%F %T'` 监控到:IP地址为:${LocalListenAddress}主机名为:`hostname` domain名为:${DOMAINNAME} 服务名:$1 监听地址:$2 监听端口:$3 的服务挂了,$5请重启之! EOF
}[ "$4" = "failed" ] && {cat < ${tf}时间:`date +'%F %T'`IP地址为:${LocalListenAddress}主机名为:`hostname` domain名为:${DOMAINNAME} 服务名:$1 监听地址:$2 监听端口:$3 的服务重启失败,请检查!!! EOF
}ftpserver ${tf} >/dev/nullrm -rf ${tf} }
cronCtrl()
{ #传过来的参数分别是:[on|off] [Admin|Managed][ $# -ne 2 ] && { echo "传递给cronCtrl函数的参数不对,请检查!;exit"}arg1=$1arg2=$2if [ "$arg1" = "off" ]then crontab -e < /dev/null 1>&2
/wls_mon.sh -a ${arg2}
:. s/^/#/g:wq!elseif [ "$arg1" = "on" ]thencrontab -e <<! 2>/dev/null 1>&2/wls_mon.sh -a ${arg2}:. s/^#//g:wq!fifi}substatus()
{ ListenAddr=$1ListenPort=$2ServerName=$3#考虑到weblogic服务周期下有多个状态,这里只是检测是否为running(虽然只有在running时才能建立起连接)/usr/local/JDK/jdk1.7.0_75/bin/java -cp ${wls_jar} weblogic.Admin -url t3://${ListenAddr}:${ListenPort} -userconfigfile ${userconfigfile} -userkeyfile ${userkeyfile} GETSTATE|grep -i running|grep -v grep > /dev/nullecho $?}GetServerStatus()
{ ListenAddr=$1ListenPort=$2ServerName=$3rt_res=2 #默认为非0，选择为2，即获取不到状态rt_res=$(substatus $1 $2 $3 &)sleep 5 #超时,这个时间请斟酌#rt_res=$? #上面语句若正常返回结果，则为0，覆盖rt_status的值if [ "${rt_res}" -eq 0 ] thenecho "0" #说明获取到的状态是running的elseif [ "${rt_res}" -eq 1 ]thenecho "1" #说明获取到的状态不是running的elseif [ "${rt_res}" -eq 2 ]thenps -ef|grep "/usr/local/JDK/jdk1.7.0_75/bin/java -cp ${wls_jar} weblogic.Admin -url t3://${ListenAddr}:${ListenPort} -userconfigfile ${userconfigfile} -userkeyfile ${userkeyfile} GETSTATE"|awk '{print "kill -9",$2}'|shecho "2" #说明java -cp进程hang住了fififi}GetProcStatus()
{ ListenAddr=$1ListenPort=$2ServerName=$3#考虑到IDS系统端口号跟服务名不匹配的情况,不以端口为关键字进程过滤ps -ef|grep -i "Dweblogic.Name=${ServerName}"|grep -v grep >/dev/null#这里加强约束,只检测java进程,像tail -f之类的进程在这里不管了echo $?}StopServer()
{ ListenAddr=$1ListenPort=$2ServerName=$3echo "`date +'%F %T'` 停服务: 服务名为 ${ServerName} 监听地址为 ${ListenAddr} 监听端口为 ${ListenPort}"#停受管服务前先判断主管服务时不时活的,如果主管挂了,则不停受管了if [ `echo $ServerName|grep Managed` ] then AdminServerName=`cat ./config/config.txt|grep -i "^AdminServer[0-9]\{1,4\}"|awk '{print $1}'` AdminListenAddr=`cat ./config/config.txt|grep -i "^AdminServer[0-9]\{1,4\}"|awk '{print $2}'` AdminListenPort=`cat ./config/config.txt|grep -i "^AdminServer[0-9]\{1,4\}"|awk '{print $3}'` if [ $(GetServerStatus $AdminListenAddr $AdminListenPort $AdminServerName) -eq 1 ] then echo "主管服务挂了,为了不影响受管服务启动,在此不停该受管了!" return 3 #退出启动进程 fifiServerStatus="$(GetServerStatus $ListenAddr $ListenPort $ServerName)"ProcStatus="$(GetProcStatus $ListenAddr $ListenPort $ServerName)"if [ ${ServerStatus} -eq 0 ] #停服务之前,先看是不是活的,如果是活的,可以用weblogic方法来停then echo "`date +'%F %T'` 服务: ${ServerName} 是活的,开始停该服务..." /usr/local/JDK/jdk1.7.0_75/bin/java -cp ${wls_jar} weblogic.Admin -url t3://${ListenAddr}:${ListenPort} -userconfigfile ${userconfigfile} -userkeyfile ${userkeyfile} FORCESHUTDOWN ${ServerName}|grep -v "^$" sleep 3 if [ ${ServerStatus} -eq 0 ] #考虑到调weblogic接口停服务并不一定成功 then echo "调用weblogic接口停服务失败,采用杀进程的方法停该服务..." ps -ef|grep ${ServerName}|grep -v grep|awk '{print "kill -9",$2}'|sh >/dev/null #这里为什么不用Dweblogic.Name？考虑到有tail -f进程,也给他杀掉 fi echo "`date +'%F %T'` 服务: ${ServerName} 已经停止!"else if [ ${ProcStatus} -eq 0 ] then echo "服务本来就挂了但是仍然有残留进程,开始清理残留进程..." ps -ef|grep ${ServerName}|grep -v grep|awk '{print "kill -9",$2}'|sh >/dev/null fi echo "`date +'%F %T'` 服务: ${ServerName} 本来就宕了!"fi }
StartServer()
{ ListenAddr=$1ListenPort=$2ServerName=$3echo "`date +'%F %T'` 启服务: 服务名为 ${ServerName} 监听地址为 ${ListenAddr} 监听端口为 ${ListenPort}"if [ $(GetServerStatus $ListenAddr $ListenPort $ServerName) -eq 0 ] #启服务前,先看服务是否启着,考虑到人工干扰then echo "`date +'%F %T'` 服务: ${ServerName} 是活的,不重新启动了!" return 3fiif [ `echo $ServerName|grep Admin` ]then if [ ! -f ${DOMAIN_DIR}/bin/startAdmin8001.sh ] then echo "路径${DOMAIN_DIR}/bin/startAdmin8001.sh 找不到,请检查!" cronCtrl on Admin #把关闭的监控任务启起来 exit else cd ${DOMAIN_DIR}/bin #注意这里为什么要这么执行,是有原因的,是为了配合脚本中清理缓存的操作 echo "`date +'%F %T'` 服务: ${ServerName} 开始启动..." #sh startAdmin8001.sh & #这样运行的话,会产生一个sh startAdmin8001.sh后台进程;会产生一个tail -f后台进程;会产生一个startWeblogic.sh后台进程,跟手工起有差别,会产生僵尸进程,不好 cat startAdmin8001.sh|grep -v "^tail -f"|grep -v "^rm " >$$.tmp sh $$.tmp rm -rf $$.tmp sleep 15 cd - >/dev/null #退出来,否则下一个序列执行bu了 fielse #启受管前要先判断主管服务是否是活的 AdminServerName=`cat ./config/config.txt|grep -i "^AdminServer[0-9]\{1,4\}"|awk '{print $1}'` AdminListenAddr=`cat ./config/config.txt|grep -i "^AdminServer[0-9]\{1,4\}"|awk '{print $2}'` AdminListenPort=`cat ./config/config.txt|grep -i "^AdminServer[0-9]\{1,4\}"|awk '{print $3}'` if [ $(GetServerStatus $AdminListenAddr $AdminListenPort $AdminServerName) -eq 1 ] then echo "主管服务挂了,请等待主管服务起来之后,再启动受管服务!" return 4 #退出启动进程 fi name=`echo $ServerName|awk '{print substr($1,length($1)-3,4)}'` if [ ! -f ${DOMAIN_DIR}/bin/startManaged${name}.sh ] then echo "路径${DOMAIN_DIR}/bin/startManaged${name}.sh 找不到,请检查!" cronCtrl on Managed #把关闭的监控任务启起来 exit else cd ${DOMAIN_DIR}/bin echo "`date +'%F %T'` 服务: ${ServerName} 开始启动..." #启动的时候获取日志 cat startManaged${name}.sh|sed -e 's/\&/2>\&1 \&/'|grep -v "^tail -f"|grep -v "wait"|grep -v "sleep 20"|grep -v "^rm " >$$.tmp sh $$.tmp rm -rf $$.tmp #这个进程必须放到后台,否则运行不下去的 tail -f ${DOMAIN_DIR}/log/${ServerName}.out >`pwd`/$$.log 2>&1 & sleep 15 cd - >/dev/null #推出来,否则下一个序列执行bu了 fifi#--重启过程中不断的检测状态count=0 ##统计监测服务状态的次数while [ $(GetServerStatus $ListenAddr $ListenPort $ServerName) -eq 1 -o $(GetProcStatus $ListenAddr $ListenPort $ServerName) -eq 1 ]do sleep ${start_interval} let count=count+1 echo "第${count}次:weblogic检测服务是否启动(返回0表示正常): $(GetServerStatus $ListenAddr $ListenPort $ServerName) 进程检测服务是否存在(返回0表示正常): $(GetProcStatus $ListenAddr $ListenPort $ServerName)" if [ $count -ge ${start_count} ] then echo "`date +'%F %T'` 启动过程中共检测服务状态${count}次,系统认为 服务: ${ServerName} 起不来!" echo "`date +'%F %T'` 服务名为 ${ServerName} 监听地址为 ${ListenAddr} 监听端口为 ${ListenPort} 重启失败,发送通知!" sendNotice $ServerName $ListenAddr $ListenPort failed [ `echo $ServerName|grep Managed` ] && { echo "`date +'%F %T'` 为了不影响系统的正常运转,将不再重启其他受管!" } #cronCtrl on Managed #把关闭的监控任务启起来 cronCtrl on $(echo ${ServerName}|cut -c1-5) #把关闭的监控任务启起来,这里是不知道是主观还是受管,故截取其前5个字符串也能实现 ps -ef|grep "tail -f ${DOMAIN_DIR}/log/$ListenPort/${ServerName}\.out.$(date +%F)"|grep -v grep|awk '{print "kill -9",$2}'|sh >/dev/null 2>&1 #为了不影响对服务的判断,杀掉该进程 rm -rf ${DOMAIN_DIR}/bin/$$.log exit fidone#看启动日志有没有报错if [ `echo $ServerName|grep Managed` ]then ps -ef|grep "tail -f ${DOMAIN_DIR}/log/$ListenPort/${ServerName}\.out.$(date +%F)"|grep -v grep|awk '{print "kill -9",$2}'|sh >/dev/null 2>&1 #为了不影响对服务的判断,杀掉该进程 cat < $$.error <Error> <.> <BEA-[0-9]{1,6}<Emergency> <.> <BEA-[0-9]{1,6}
<Critical> <.*> <BEA-[0-9]{1,6}EOFcat ${DOMAIN_DIR}/bin/$$.log|grep -i -f $$.error >/dev/nullif [ $? -eq 0 ]thenecho "date +'%F %T' 检查到服务在启动时报错,将该受管进程杀掉!"ps -ef|grep -i ${ServerName}|grep -v grep|awk '{print "kill -9",$2}'|sh >/dev/nullecho "date +'%F %T' 服务: ${ServerName} 启动失败,等待监控时再自动重启!" echo "`date +'%F %T'` 服务名为 ${ServerName} 监听地址为 ${ListenAddr} 监听端口为 ${ListenPort} 重启失败,发送通知!" sendNotice $ServerName $ListenAddr $ListenPort failed echo "`date +'%F %T'` 为了不影响系统的正常运转,将不再重启其他受管!" #cronCtrl on Managed #把关闭的监控任务启起来,而且这里将日志写完 echo "" echo "`date +'%F %T'` 重启***受管服务***任务执行完毕!" #有可能是定时重启,也有可能是监控重启 echo "`date +'%F %T'` 将wls_mon.sh监控任务起来" echo "******************************************`date`******************************************" cronCtrl on Managed #如果是监控时重启失败的话,监控前并没有关闭crontab,也就是说没有必要开启 rm -rf $$.error >/dev/null rm -rf ${DOMAIN_DIR}/bin/$$.log exit #退出进程 #删除启动临时日志 else echo "`date +'%F %T'` 服务: ${ServerName} 启动完毕!" fi rm -rf $$.error >/dev/nullelse echo "`date +'%F %T'` 服务: ${ServerName} 启动完毕!"firm -rf ${DOMAIN_DIR}/bin/$$.log }
MonAdminServer() ##开始循环监控主管
{ #传过来的参数分别是:服务名称 监听地址 监听端口[ $# -ne 3 ] && { echo "传递给MonAdminServer函数的参数不对,请检查!;exit"}ServerStatus="$(GetServerStatus $2 $3 $1)"ProcStatus="$(GetProcStatus $2 $3 $1)"if [ ${ServerStatus} -eq 1 -a ${ProcStatus} -eq 1 ]then echo "`date +'%F %T'` 监控到:服务名:$1 监听地址:$2 监听端口:$3 的服务挂了,重启之!" StartServer $2 $3 $1else if [ ${ServerStatus} -eq 1 -a ${ProcStatus} -eq 0 ] then echo "`date +'%F %T'` 监控到:服务名:$1 监听地址:$2 监听端口:$3 的服务挂了,但是进程还存在,杀掉进程,重启之!" ps -ef|grep $1 |grep -v grep|awk '{print "kill -9",$2}'|sh >/dev/null StartServer $2 $3 $1 else if [ ${ServerStatus} -eq 0 ] then echo "`date +'%F %T'` 监控到:服务名:$1 监听地址:$2 监听端口:$3 的服务是活的!" fi fifi }
MonManagedServer() ##开始循环监控受管
{ #传过来的参数分别是:服务名称 监听地址 监听端口[ $# -ne 3 ] && { echo "传递给MonManagedServer函数的参数不对,请检查!;exit"}ServerStatus="$(GetServerStatus $2 $3 $1)"ProcStatus="$(GetProcStatus $2 $3 $1)"if [ ${ServerStatus} -eq 1 -a ${ProcStatus} -eq 1 ]then if [ `date +'%H'` -ge 0 -a `date +'%H'` -le 6 ] #若时间在凌晨12点到6点之间 then echo "`date +'%F %T'` 监控到:服务名:$1 监听地址:$2 监听端口:$3 的服务挂了,重启之!" StartServer $2 $3 $1 else echo "`date +'%F %T'` 监控到:服务名:$1 监听地址:$2 监听端口:$3 的服务挂了,发送通知!" sendNotice $1 $2 $3 down fielse if [ ${ServerStatus} -eq 1 -a ${ProcStatus} -eq 0 ] then if [ `date +'%H'` -ge 0 -a `date +'%H'` -le 6 ] then echo "`date +'%F %T'` 监控到:服务名:$1 监听地址:$2 监听端口:$3 的服务挂了,但是进程还存在,杀掉进程,重启之!" ps -ef|grep $1 |grep -v grep|awk '{print "kill -9",$2}'|sh >/dev/null StartServer $2 $3 $1 else echo "`date +'%F %T'` 监控到:服务名:$1 监听地址:$2 监听端口:$3 的服务挂了,但是进程还存在,发送通知!" sendNotice $1 $2 $3 down "但是进程还存在," fi else if [ ${ServerStatus} -eq 0 ] then echo "`date +'%F %T'` 监控到:服务名:$1 监听地址:$2 监听端口:$3 的服务是活的!" else if [ ${ServerStatus} -eq 2 ] then echo "`date +'%F %T'` 监控到:服务名:$1 监听地址:$2 监听端口:$3 的服务是hang住了，发送通知!" sendNotice $1 $2 $3 down "服务hang住了," fi fi fifi }
ChkEnv()
{ #环境检查if [ ! -f ./config/config.txt ]thenecho "pwd/../config/config.txt文件不存在,请检查!"exitelse#读取config中环境变量cat ./config/config.txt|grep -i "^export" >SetEnv.sh.$$ #这里用$$区分,否则多个进程间会打架. ./SetEnv.sh.$$rm -rf SetEnv.sh.$$fi [ ! -f ${wls_jar} ] && {echo "weblogic.jar不存在,请检查/home/weblogic/weblogic12/wlserver/server/lib下是否有weblogic.jar"exit}[ ! -f ${DOMAIN_DIR}/config/config.xml ] && {echo "${DOMAIN_DIR}/config/config.xml文件不存在!"exit}[ ! -d ${DOMAIN_DIR}/bin ] && {echo "${DOMAIN_DIR}/bin目录不存在!"exit}[ ! -f ${userconfigfile} -o ! -f ${userkeyfile} ] && {echo "存储weblogic用户名和密码的userconfigfile和userkeyfile不存在,请检查`pwd`/config/下是否存在该文件"echo "参考命令: /usr/local/JDK/jdk1.7.0_75/bin/java -cp /home/weblogic/weblogic12/wlserver/server/lib/weblogic.jar -Duser.home=`pwd` -Duser.name=`hostname` weblogic.Admin -username weblogic -password weblogic STOREUSERCONFIG"exit}[ -z "$(cat config/config.txt|grep -e "^ManagedServer[0-9]\{1,4\}")" ] && {echo "`pwd`/config/config.txt中没有domain的***受管***信息,或者受管信息被#屏蔽了,请检查!"exit}[ -z "$(cat config/config.txt|grep -e "^AdminServer[0-9]\{1,4\}")" ] && {echo "`pwd`/config/config.txt中没有domain的***主管***信息,请检查!"exit}[ "${IsExistAdminServer}" = "true" -a ! -f ${cronrebAdminlog} ] && {touch ${cronrebAdminlog}}[ "${IsExistAdminServer}" = "true" -a ! -f ${monAdminlog} ] && {touch ${monAdminlog}}[ ! -f ${cronrebManagedlog} ] && {touch ${cronrebManagedlog}}[ ! -f ${monManagedlog} ] && {touch ${monManagedlog}} }
ChkEnv
case $1 in
-c*)shift 1#定时重启主管并且本domain确实存在主管if [ "$1" = "Admin" -a "${IsExistAdminServer}" = "true" ] thenChkEnvecho "**date**" >>${cronrebAdminlog}echo "date +'%F %T' 定时重启主管服务任务开始执行!" >>${cronrebAdminlog}echo "" >>${cronrebAdminlog}echo "date +'%F %T' 先把crontab中监控主管状态的任务屏蔽,防止冲突!" >>${cronrebAdminlog}cronCtrl off Admin >>${cronrebAdminlog}cat ./config/config.txt|grep -i "^AdminServer[0-9]{1,4}"|while read ServerName ListenAddr ListenPortdoStopServer ${ListenAddr} ${ListenPort} ${ServerName} >>${cronrebAdminlog} #停服务StartServer ${ListenAddr} ${ListenPort} ${ServerName} >>${cronrebAdminlog} #起服务doneecho "" >>${cronrebAdminlog}echo "date +'%F %T' 定时重启主管服务任务执行完毕!" >>${cronrebAdminlog}echo "date +'%F %T' 将wls_mon.sh监控任务起来">>${cronrebAdminlog}echo "**date**" >>${cronrebAdminlog}sleep 60cronCtrl on Admin >>${cronrebAdminlog}else if [ "$1" = "Managed" ]thenChkEnvecho "**date**" >>${cronrebManagedlog}echo "date +'%F %T' 定时重启受管服务任务开始执行!" >>${cronrebManagedlog}echo "date +'%F %T' 先把crontab中监控受管状态的任务屏蔽,防止冲突!" >>${cronrebManagedlog}cronCtrl off Managed >>${cronrebManagedlog}cat ./config/config.txt|grep -i "^ManagedServer[0-9]{1,4}"|while read ServerName ListenAddr ListenPortdoecho "" >>${cronrebManagedlog}echo "*服务名:${ServerName} 监听地址:${ListenAddr} 监听端口:${ListenPort}***" >>${cronrebManagedlog}StopServer ${ListenAddr} ${ListenPort} ${ServerName} >>${cronrebManagedlog}StartServer ${ListenAddr} ${ListenPort} ${ServerName} >>${cronrebManagedlog}doneecho "" >>${cronrebManagedlog}echo "date +'%F %T' 定时重启受管服务任务执行完毕!" >>${cronrebManagedlog}echo "date +'%F %T' 将wls_mon.sh监控任务起来">>${cronrebManagedlog}echo "**date**" >>${cronrebManagedlog}sleep 60cronCtrl on Managed >>${cronrebManagedlog}elseecho "执行定时重启主/受管的参数错误!"usagefifi;;-a*)shift 1#监控主管并且本domain确实存在主管if [ "$1" = "Admin" -a "${IsExistAdminServer}" = "true" ]thenChkEnvecho "**date**" >>${monAdminlog}echo "date +'%F %T' 监控主管服务任务开始执行!" >>${monAdminlog}echo "" >>${monAdminlog}cat ./config/config.txt|grep -i "^AdminServer[0-9]{1,4}"|while read ServerName ListenAddr ListenPortdoMonAdminServer ${ServerName} ${ListenAddr} ${ListenPort} >>${monAdminlog}doneecho "" >>${monAdminlog}echo "date +'%F %T' 监控主管服务任务执行完毕!" >>${monAdminlog}echo "**date**" >>${monAdminlog}elseif [ "$1" = "Managed" ]thenChkEnvecho "**date**" >>${monManagedlog}echo "date +'%F %T' 监控受管服务任务开始执行!" >>${monManagedlog}echo "" >>${monManagedlog}cat ./config/config.txt|grep -i "^ManagedServer[0-9]{1,4}"|while read ServerName ListenAddr ListenPortdoMonManagedServer ${ServerName} ${ListenAddr} ${ListenPort} >>${monManagedlog}doneecho "" >>${monManagedlog}echo "date +'%F %T' 监控受管服务任务执行完毕!" >>${monManagedlog}echo "**date**" >>${monManagedlog}elseecho "执行定时重启主/受管的参数错误!"usagefifi;;*)usage;;esac 最后将脚本命令添加到计划任务中即可!
转载于:https://blog.51cto.com/11612019/2156667