svn备份、ftp传输脚本

你可以使用resync来保持两台机器的数据同步(不建议小文件变更多、快情况下使用)
系统:centos 5.6
svn version:1.6.11
是否需要mail服务:是,请安装sendmail、mutt(可选 只要你linux那可以发送邮件就可以)
是否与http服务整合:是
里面内容请参考如下两处文章

FTP备份用的一个Shell方法函数 http://kinggoo.com/ftp-ftpfunction.htm
svn服务增量备份脚本,Incremental backup script http://kinggoo.com/svn-incrementalscript.htm

[root@kinggoo.com cron]# cat svnBackup.cron.sh

#!/bin/bash
####  http://kinggoo.com
####  需要安装sendmail、mutt以便发送邮件通知,当然也可以改下代码,弄成网页访问形式也可以
####  不要删除SVN_LOG后面设置变量的任何文件,否则备份时会自动全备
####  if [[ ${WEEKDAY} = 'Fri' ]];then 这里是做全备的判断可以修改为其他,但请尽量不要修改为周末。
TIME=`date +%Y%m%d`
YM=`date +%Y%m`
WEEKDAY=`date +%a`
DAYTIME=`date +%Y%m%d%H`
SVNROOT=/opt/svnroot
SVNDATA=/svn
HTTP=/etc/init.d
#请创建用于全备的目录文件SVNALL
SVNALL=${SVNDATA}/svnall
SVN_LOG=${SVNDATA}/log
TOMAIL="kinggoo@kinggoo.com"
if [ ! -e ${SVNDATA}/${YM} ] || [ ! -d ${SVNDATA}/${YM} ];then
        mkdir -p ${SVNDATA}/${YM}
        SVNDATA=${SVNDATA}/${YM}
else
        SVNDATA=${SVNDATA}/${YM}
fi
echo "" > ${SVN_LOG}/svn.mail > /dev/null
#针对配置http集成svn服务来做,如果你没有将http添加到服务。脚本内是通过http服务生效后返回显示信息OK来判断是否启动的!
declare -a  REPOSITORY
REPOSITORY=( KingGoo com )
ARR=0
# - function 集合
function RunStophttp()
{         # apache服务起停
                        ${HTTP}/httpd $1 |grep -i OK && I=$? || I=1
                        sleep 5
                        if [ "$1" = "stop" ];then
                                if [[ "${I}" -eq "0" ]];then
                                        echo "-- httpd is stopping" >> ${SVN_LOG}/svn.mail
                                else
                                        /usr/bin/pgrep httpd|xargs kill -9
                                        echo "-- httpd is kill for stopping" >> ${SVN_LOG}/svn.mail
                                fi
                        else
                                if [ "$1" = "start" ];then
                                        if [[ "${I}" -eq "0" ]];then
                                                echo "-- httpd is running" >> ${SVN_LOG}/svn.mail
                                        else
                                                echo "-- httpd is failed" >> ${SVN_LOG}/svn.mail
                                                ${HTTP}/httpd restart |grep -i OK && I=$? || I=ERROR
                                                echo -e "成功:0,失败:1 -- \r\n [ $I ] " >> ${SVN_LOG}/svn.mail
                                        fi
                                fi
                        fi
                }

##function end
# ---------------------------

while [ "${ARR}" -lt "${#REPOSITORY[@]}" ]
do
        if [ ! -e ${SVN_LOG} ] || [ ! -d ${SVN_LOG} ];then
                mkdir -p ${SVN_LOG}
        fi
        if [ ! -e ${SVNALL} ] || [ ! -d ${SVNALL} ];then
                mkdir -p ${SVNALL}
        fi
        if [ ! -e ${SVNDATA}/${REPOSITORY[$ARR]} ] || [ ! -d ${SVNDATA}/${REPOSITORY[$ARR]} ];then
                mkdir -p ${SVNDATA}/${REPOSITORY[$ARR]}
        fi
        BUILD=`svnlook  youngest ${SVNROOT}/${REPOSITORY[$ARR]}` && sleep 2
        KG_BUILD=`cat ${SVN_LOG}/${REPOSITORY[$ARR]}.version`
        if [ ! -e ${SVN_LOG}/${REPOSITORY[$ARR]}.version ] && [ -z ${KG_BUILD} ];then
                echo "0" > ${SVN_LOG}/${REPOSITORY[$ARR]}.version
                KG_BUILD=0
        else
                #将最新得到的reversion存储起来,并将之前reversion+1
                ((KG_BUILD=${KG_BUILD}+1))
                echo "${BUILD}" > ${SVN_LOG}/${REPOSITORY[$ARR]}.version
        fi
              
        #ago 小于等于 after 
        if [ ${KG_BUILD} -le ${BUILD} ];then
                RunStophttp stop 
                if [[ ${WEEKDAY} = 'Fri' ]];then 
				##为什么是周五,原因在周末做全备此脚本是不可能的,当然你可以改下我里面的判断逻辑是可以改的。但发这个版本我并没有时间去做修改了,希望理解!
                        svnadmin dump ${SVNROOT}/${REPOSITORY[$ARR]}  > ${SVNALL}/backup.all.${REPOSITORY[$ARR]}.${BUILD}.${DAYTIME}.dump  2> ${SVNALL}/log.${DAYTIME} && sleep 30
                        echo -e "已执行全备信息如下:\r\n `ls -l ${SVNALL}/backup.all.${REPOSITORY[$ARR]}.${BUILD}.${DAYTIME}.dump`" >>  ${SVN_LOG}/svn.mail
                fi
                echo "${REPOSITORY[$ARR]}:${YM}:${KG_BUILD}:${BUILD}" >> ${SVN_LOG}/svn.important
                svnadmin dump ${SVNROOT}/${REPOSITORY[$ARR]} -r ${KG_BUILD}:${BUILD} --incremental > ${SVNDATA}/${REPOSITORY[$ARR]}/${REPOSITORY[$ARR]}.${KG_BUILD}_${BUILD}.dump.${TIME} 2>> ${SVN_LOG}/${REPOSITORY[$ARR]}.log && sleep 30
                echo -e "已备份${REPOSITORY[$ARR]}代码库 \r\n `ls -l ${SVNDATA}/${REPOSITORY[$ARR]}/${REPOSITORY[$ARR]}.${KG_BUILD}_${BUILD}.dump.${TIME}`" >>  ${SVN_LOG}/svn.mail
                RunStophttp start
        else
                echo -e "代码库:${REPOSITORY[$ARR]} , 前一版本:${KG_BUILD} 大于后一版本:${BUILD} ,没有执行备份操作!" >> ${SVN_LOG}/svn.mail
        fi
        ((ARR=$ARR+1))
        unset KG_BUILD BUILD BUILD_TEMP
done
echo "`cat ${SVN_LOG}/svn.mail`" |mutt -s "Svn incremental backup ${TIME}" ${TOMAIL}
/var/spool/cron/ftpRemote.cron.sh > /root/svn.ftp.log 2>&1

FTP备份用的一个Shell方法函数详解

[root@kinggoo cron]# cat ftpRemote.cron.sh
其实可以用传参来做,但我们这每天备份都是有规律的,所以直接用最新文件来标记

#!/bin/bash
## function: upload to remote 
# $1:本地文件路径+文件名,$2:远程目录
function funFtp()
{
CDLOAD="`dirname $1`";FTP_SNAME="`basename $1`";FTP_RNAME="$2"
/usr/bin/ftp -n<<!
open ${FTP_IP}
user ${FTP_USER}  ${FTP_PASSWD}
prompt
lcd ${CDLOAD}
cd ${FTP_RNAME}
mput ${FTP_SNAME}
close
bye
!
}

## variable
BACK="/back/"
SLEEP="sleep 5"
SVN_LOG="/svn/log"
FTP_IP="ftp服务器地址"
FTP_USER="backup"  #ftp访问的用户名
FTP_PASSWD="backupdate" #ftp访问的密码
TIME=`date  +%Y%m%d`
# # #ls -t file|head -1 是获取最新的第一个文件
# # atlassian variable
# # 例子jira
# DATA_NAME="/opt/local/DATA_HOME/"
# JIRA_NAME=`ls -t ${DATA_NAME}Jira/export/*.zip | head -1` 获取最新
# WIKI_NAME=`ls -t ${DATA_NAME}Confluence/backups/*.zip | head -1`
# BACK_JIRA="${BACK}/atlassian.jira.attachments.${TIME}.tar.gz"
# JIRA_FILE_1="${DATA_NAME}Jira/data/attachments"

# BACK_WIKI="${BACK}/atlassian.confluence.attachments.${TIME}.tar.gz"
# WIKI_FILE_1="${DATA_NAME}Confluence/attachments"

# # testlink例子
# # testlink variable
# TL_NAME="testlink.allfile.${TIME}.tar.gz"
# TL_FILE_1="/var/www/html/TL"


# # svn variable
# 用数组来保存,方便后面循环,如果你有多个库可以多写些
# SVN_ALLNAME 全备份后的文件
# SVN_INCREMENTNAME 增量备份的文件
declare -a SVN_ALLNAME SVN_INCREMENTNAME
SVN_INCREMENTNAME=( `ls -t /svn/*/*/KingGoo.*.dump.*|head -1` `ls -t  /svn/*/*/Vancl.*.dump.*|head -1` `ls -t /svn/*/*/Vjia.*.dump.*|head -1` )
SVN_ALLNAME=( `ls -t /svn/svnall/backup.all.KingGoo*.dump|head -1` `ls -t /svn/svnall/backup.all.Vancl*.dump|head -1` `ls -t /svn/svnall/backup.all.Vjia*.dump|head -1` )

# # atlassian backup
# cd  `dirname ${JIRA_FILE_1}`
# tar -zcvf  ${BACK_JIRA} ./attachments  && ${SLEEP} 
# cd `dirname ${WIKI_FILE_1}` 
# tar -zcvf ${BACK_WIKI} ./attachments && ${SLEEP}
# cp ${JIRA_NAME} ${BACK}atlassian.jira.`basename ${JIRA_NAME}` && ${SLEEP}
# cp ${WIKI_NAME} ${BACK}atlassian.confluence.`basename ${WIKI_NAME}` && ${SLEEP}
# funFtp "${BACK}atlassian.jira.`basename ${JIRA_NAME}`" "atlassian"
# funFtp "${BACK}atlassian.confluence.`basename ${WIKI_NAME}`"  "atlassian"
# funFtp "${BACK_JIRA}"  "atlassian"
# funFtp "${BACK_WIKI}"  "atlassian"
# # testlink backup
# cd ${TL_FILE_1}
# tar -zcvf ${BACK}${TL_NAME} ./* --exclude=./logs/* --exclude=./install/* --exclude=./docs/*  && ${SLEEP}
# funFtp "${BACK}${TL_NAME}" "testlink" &&  ${SLEEP}
# # openldap backup
# /usr/sbin/slapcat -l ${BACK}openldap.${TIME}.ldif && ${SLEEP}
# funFtp "${BACK}openldap.${TIME}.ldif" "openldap" && sleep 2
# svn backup
i=0
while [ "${i}" -lt "${#SVN_INCREMENTNAME[@]}" ]
do
        if [[ ! -z ${SVN_INCREMENTNAME[$i]} ]];then
        cp ${SVN_INCREMENTNAME[$i]} ${BACK} && sleep 2
        funFtp "${BACK}`basename ${SVN_INCREMENTNAME[$i]}`" "svn" &&sleep 2
        fi
((i=$i+1))
done
i=0
while [ "${i}" -lt "${#SVN_ALLNAME[@]}" ]
do
        if [[ ! -z  ${SVN_ALLNAME[$i]} ]] ;then
        mv ${SVN_ALLNAME[$i]}  ${BACK} && sleep 2
        funFtp "${BACK}`basename ${SVN_ALLNAME[$i]}`" "svn/svnall" && sleep 5
        fi
((i=$i+1))
done
#邮箱,还有一些东西都可以用参数形式传过来。我其实当时想写成引入变量方式的,但没时间修改。
echo "backup " | mutt -s "mmsg" kinggoo@kinggoo.com

之前忘记把定期清理的任务写到文章里了,你可以设置为,每天或者每周都可以,当尽量不要太久才清理一次

#每天三点执行下面信息(记得如果路径你使用变量写到脚本里了,千万要把删除的这条命令放到unset的前面)
#ftp端也要写定期清理的任务
0 3 * * * find /back/ -ctime +7 -exec rm -f {} \;

希望有帮助,有问题及建议请留言,或微博我!联系方式请访问我的关于界面。

[20130408] – 增加忘记写到文章中的定期清理备份数据任务

- THE END -
版权声明:
转载原创文章请注明,文章出处:http://kinggoo.com
原文地址:http://kinggoo.com/svnincrementalftp.htm
发表评论?

8 条评论。

  1. 我是将ftp自动上传的这段脚本,直接加在全量备份脚本的下面的。
    我查看了全量备份完成的时间和dump上传的时间,发现一个问题:
    全量备份还没有结束,那边的自动上传就开始执行了。我以为要等到备份结束,才会执行FTP上传

    • 估计你是改脚本之后,把类是这个功能的语句内的&&去掉了,他的意思是完全执行完前面的并且正确才会执行后面的语句 。如果 是没备份完就去执行 ftp了 那估计就是这个问题
      svnadmin dump ${SVNROOT}/${REPOSITORY[$ARR]} -r ${KG_BUILD}:${BUILD} –incremental > ${SVNDATA}/${REPOSITORY[$ARR]}/${REPOSITORY[$ARR]}.${KG_BUILD}_${BUILD}.dump.${TIME} 2>> ${SVN_LOG}/${REPOSITORY[$ARR]}.log && sleep 30

      • 我现在全量备份脚本中,是如下这样的:
        svnadmin dump ${xxx}/hgj > ${SVN_DATA}/hgj_$TIME.dump 2> ${SVN_DATA}/svn.hgj.build.version && sleep 300 & svnadmin dump ${xxx}/nb2 > ${SVN_DATA}/nb2_$TIME.dump 2> ${SVN_DATA}/svn.nb2.build.version && sleep 300 & svnadmin dump ${xxx}/ft_bbp > ${SVN_DATA}/ft_bbp_$TIME.dump 2> ${SVN_DATA}/svn.ft_bbp.build.version && sleep 300 & svnadmin dump ${xxx}/cxzt > ${SVN_DATA}/cxzt_$TIME.dump 2> ${SVN_DATA}/svn.cxzt.build.version && sleep 300 & svnadmin dump ${xxx}/nfscloud > ${SVN_DATA}/nfscloud_$TIME.dump 2> ${SVN_DATA}/svn.nfscloud.build.version && sleep 10 && sleep 10 && /etc/init.d/httpd start |grep -i OK && I=$? ;

        ftp的命令,就加在了这个的下面

  2. 请教一个问题:
    我的FTP是这样写的
    ftp -n<<!
    open 192.168.213.250
    user yuanjie yuanjie
    binary
    hash
    cd /opt/svn_backup/192.168.201.15/weekly_backup
    lcd /home/svn_backup/weekly_backup
    prompt
    mput hgj_$TIME.dump nb2_$TIME.dump ft_bbp_$TIME.dump cxzt_$TIME.dump nfscloud_$TIME.dump
    close
    bye
    !

    要上传5个全量备份dump文件,但每次上传都出现大容量的dump文件才上传20分之一就自动中断了。这5个dump文件,最大的是16.5g,最小的是98M

    下面是vsftpd.log中的信息:
    Thu Jun 27 14:30:45 2013 68 192.168.201.15 2450758348 /opt/svn_backup/192.168.201.15/weekly_backup/hgj_20130627.dump b _ i r yuanjie ftp 0 * c
    Thu Jun 27 14:30:49 2013 4 192.168.201.15 60935504 /opt/svn_backup/192.168.201.15/weekly_backup/nb2_20130627.dump b _ i r yuanjie ftp 0 * c
    Thu Jun 27 14:30:57 2013 8 192.168.201.15 75443605 /opt/svn_backup/192.168.201.15/weekly_backup/ft_bbp_20130627.dump b _ i r yuanjie ftp 0 * c
    Thu Jun 27 14:30:58 2013 1 192.168.201.15 97684272 /opt/svn_backup/192.168.201.15/weekly_backup/cxzt_20130627.dump b _ i r yuanjie ftp 0 * c
    Thu Jun 27 14:30:59 2013 1 192.168.201.15 44967103 /opt/svn_backup/192.168.201.15/weekly_backup/nfscloud_20130627.dump b _ i r yuanjie ftp 0 * c

发表评论