mongodb 备份压缩及清除指定日期前的数据

2016-12-29 09:54:57来源:oschina作者:空灵飞渡人点击

第七城市

#!/usr/bin/bash


#define:Mongodb data backup regularly, delete data before the specified time


#date:2016-12-20


curr_dir=`pwd` back_dir="/data/bak/mongodb_bak" mongo_comm="/usr/local/mongodb/bin/mongo" mongo_dump="/usr/local/mongodb/bin/mongodump" mongo_data=" robot engine edi"


####rsync同步信息 rsync_user=*** rsync_passwd=**** echo "$rsync_passwd" > $curr_dir/rsync_passwd passwd="$curr_dir/rsync_passwd" rsync_server=**** rsync_module=mongo_com


#####mongodb各库需要保存的天数 #robot_delete_day=`date -d "25 days ago" +"%Y-%m-%d %H:%M:%S"` robot_delete_day=`date -d "24 days ago" +"%Y-%m-%d"` #engine_delete_day=`date -d "60 days ago" +"%Y-%m-%d %H:%M:%S"` engine_delete_day=`date -d "13 days ago" +"%Y-%m-%d"` #edi_delete_day=`date -d "16 days ago" +"%Y-%m-%d %H:%M:%S"` edi_delete_day=`date -d "52 days ago" +"%Y-%m-%d"` today=`date +"%Y%m%d"` transferr_data_dir="/data/bak/mongo_data_back/$today"


id_dir="/data/bak/mongo_id/$today" [[ -d $id_dir ]]|| mkdir -p $id_dir


logdir="/data/log/mongodb_delete_log" #日志路径 log=$logdir/log"$today".log#日志文件 is_font=1 #终端是否打印日志: 1打印 0不打印 is_log=1#是否记录日志: 1记录 0不记录 [[ -d $logdir ]]||mkdir -p $logdir#确保日志目录存在


datef(){ date "+%Y-%m-%d %H:%M:%S" }


print_log(){ if [[ $is_log -eq 1 ]];then [[ -d $logdir ]] || mkdir -p $logdir echo -e "[ $(datef) ] $1" >> $log fi if [[ $is_font -eq 1 ]];then echo -e "[ $(datef) ] $1" fi }


####mongodb数据定期备份、压缩、同步至大数据存储 mongodb_back_regualy() { for data in $mongo_data doprint_log "判断备份目录是否存在"[[ -d $back_dir/$today/$data ]]|| mkdir -p $back_dir/$today/$datawhile truedo print_log "$data 开始备份" $mongo_dump -d $data -o $back_dir/$today/$data if [[ $? -eq 0 ]];thenprint_log "$data 备份成功"break elseprint_log "$data 备份失败,继续" fidonecd $back_dir/$today/$datawhile truedo print_log "$data 开始压缩" [[ -d $transferr_data_dir ]]|| mkdir -p $transferr_data_dir tar -zcf $transferr_data_dir/"$data"_"$today".tar.gz "$data" if [[ $? -eq 0 ]];thenprint_log "'$data'_'$today' 压缩成功"break elseprint_log "'$data'_'$today' 压缩失败,继续压缩" fidonewhile truedo print_log "'$data'_'$today'.tar.gz 开始传输" #####rsync传输,传输目录变量待修改 #rsync -avzr --password-file=$passwd $transferr_data_dir $rsync_user@$rsync_server::$rsync_module if [[ $? -eq 0 ]];thenprint_log "'$data'_'$today'.tar.gz 传输成功"break elseprint_log "'$data'_'$today'.tar.gz 传输失败" fidone


done }


mongo_robot_delete() {print_log "开始统计robot库总条数"mongo_sql_sum='db.robotTask.find().count()'mongo_sql_sum_cal=`echo "$mongo_sql_sum"|$mongo_comm $data --shell|grep ^[0-9]`print_log "robot库总条数为------$mongo_sql_sum_cal-------"#print_log "开始统计robot库要删除的条数" while truedo mongo_sql_count='db.robotTask.find({"startTime":{$lt:ISODate("'$robot_delete_day'")}}).count()' mongo_sql_count_cal=`echo "$mongo_sql_count"|$mongo_comm $data --shell` need_delete_total=`echo "$mongo_sql_count_cal"|grep ^[0-9]` if [[ $need_delete_total -le 0 ]];thenprint_log "robot 删除完毕"break elseprint_log "robot库需要删除的条数为 $need_delete_total --------------------------"mongo_data_delete='db.robotTask.remove({"startTime":{$lt:ISODate("'$robot_delete_day'")}})'echo "$mongo_data_delete" | $mongo_comm $data --shell fidone }


mongo_engine_delete() {print_log "开始统计engine库总条数"mongo_sql_sum='db.rulemessage.find().count()'mongo_sql_sum_cal=`echo "$mongo_sql_sum"|$mongo_comm $data --shell|grep ^[0-9]`print_log "engine库总条数为--------$mongo_sql_sum_cal---------"#print_log "开始统计engine库要删除的条数"while truedo mongo_sql_count='db.getCollection("rulemessage").find({"lastUpdated":{$lte:new Date("'$engine_delete_day'").getTime()}}).count()' mongo_sql_count_cal=`echo "$mongo_sql_count"|$mongo_comm $data --shell` need_delete_total=`echo "$mongo_sql_count_cal"|grep ^[0-9]` if [[ $need_delete_total -le 0 ]];thenprint_log "engine 删除完毕"break elseprint_log "engine库需要删除的条数为 $need_delete_total --------------------------"mongo_data_delete='db.rulemessage.remove({"lastUpdated":{$lte:new Date("'$engine_delete_day'").getTime()}})'echo "$mongo_data_delete" | $mongo_comm $data --shell fidone }


mongo_edi_delete() {print_log "开始统计edi库总条数"mongo_sql_sum='db.action_log.find().count()'mongo_sql_sum_cal=`echo "$mongo_sql_sum"|$mongo_comm $data --shell |grep ^[0-9]`print_log "edi库总条数为-------$mongo_sql_sum_cal---------"#print_log "开始统计edi库要删除的条数"while truedo mongo_sql_count='db.action_log.find({"sendTime":{$lt:ISODate("'$edi_delete_day'")}}).count()' mongo_sql_count_cal=`echo "$mongo_sql_count"|$mongo_comm $data --shell` need_delete_total=`echo "$mongo_sql_count_cal"|grep ^[0-9]` if [[ $need_delete_total -le 0 ]];thenprint_log "edi 删除完毕"break elseprint_log "edi库需要删除的条数为 $need_delete_total --------------------------"mongo_data_delete='db.action_log.remove({"sendTime":{$lt:ISODate("'$edi_delete_day'")}})'echo "$mongo_data_delete" | $mongo_comm $data --shell fidone }


data_delete(){cd $id_dirprint_log "删除各库10天以前的id数据"find ./ -mtime +10 |xargs rm -rfprint_log "删除各库备份两天以前的数据"cd $back_dirfind ./ -mtime +1 |xargs rm -rfprint_log "删除各库压缩4天以前的数据"cd $transferr_data_dirfind ./ -mtime +3 |xargs rm -rf


}


mongodb_back_regualy


for data in $mongo_data docase $data inrobot) mongo_robot_delete;;engine) mongo_engine_delete;;edi) mongo_edi_delete;;*) echo "$1" echo "Usage:集合不存在";;esac


done


data_delete

第七城市

最新文章

123

最新摄影

微信扫一扫

第七城市微信公众平台