读取es数据,生成html表格发送邮件

2018-02-27 11:18:29来源:oschina作者:雪地花露人点击

分享
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import requests
import json,MySQLdb
import codecs
import os,subprocess
from collections import Counter
from email.header import Header
from email.mime.text import MIMEText
from email.utils import parseaddr, formataddr
import smtplib
import datetime,time
from elasticsearch import Elasticsearch
url = "http://***:9200/"
index_name = "deploy_metrics_pro"
# 实例化Elasticsearch类,并设置超时间为120秒,默认是10秒的,如果数据量很大,时间设置更长一些
es = Elasticsearch(url,timeout=120)
# DSL查询语法,在下面es.search使用
sre_user = ['***','**']
defchange(x):
ifx < 10:
return '0{}'.format(x)
else:
return x
data = {
"size": 10000000, #指定每个分片最大返回的数据量,可根据日志量进行设置
"query" : {
"bool":{
# 指定要匹配的字符,这里是查找所有数据
"must" : {
"term":{'type':'deployment'}
},
"filter":{
"range":{
"start_time_format":{
"gt": "%s-%s-%s 19:00:00" %(time.localtime().tm_year,change(time.localtime().tm_mon),change(time.localtime().tm_mday-7)),
"lt": "%s-%s-%s 19:00:00" %(time.localtime().tm_year,change(time.localtime().tm_mon),change(time.localtime().tm_mday)),
}
},
},
}
}
}
# 设置要过滤返回的字段值,要什么字段,就在这里添加,这样可以节约返回的数据量(带宽,内存等)
return_fields = [
'_scroll_id',
'hits.hits._source.timestamp',
'hits.hits._source.[@timestamp](https://my.oschina.net/u/1049939)',
'hits.hits._source.project',
'hits.hits._source.username',
'hits.hits._source.start_time_format',
'hits.hits._source.department_name',
'hits.hits._source.project_type',
'hits.hits._source.type',
]
def es_data():
# 指定search_type="scan"模式,并返回_scroll_id给es.scroll获取数据使用
d = []
res = es.search(
index=index_name,
body=data,
search_type="scan",
scroll="1m"
)
scrollId=res["_scroll_id"]# 获取scrollID
response= es.scroll(scroll_id=scrollId, scroll= "1m",filter_path=return_fields,)
#print response
#print len(response['hits']['hits']) # 打印获取到的日志数量
for hit in response['hits']['hits']:
d.append(hit['_source'])
return d
def get_data():
con1 = {}
con2 = {}
for i in es_data():
#print i
if i['type']=='deployment' and i['project_type']==0:
#print i['department_name'],i['name'],i['op_user']
if con1.has_key(i['department_name']):
con1[i['department_name']].append(i['project'])
else:
con1[i['department_name']] = []
if con2.has_key(i['department_name']):
con2[i['department_name']].append(i['username'])
else:
con2[i['department_name']] = []
return [con1,con2]
def get_list(flag):
con = get_data()
d = []
if flag == 'department':
for k in con[0].keys():
c = {}
c['department'] = k
c['count'] = len(con[0][k])
d.append(c)
return d
elif flag == 'depart-pro':
for k in con[0].keys():
s = Counter(con[0][k]).most_common(10)
for i in s:
c = {}
c['department'] = k
c['name'] = i[0]
c['count'] = i[1]
# print k,c
d.append(c)
#print d
return d
else:
for k in con[1].keys():
s = Counter(con[1][k]).most_common()
for i in s:
if i[0] not in sre_user:
c = {}
c['department'] = k
c['name'] = i[0]
c['count'] = i[1]
d.append(c)
return d
def render_data(data,flag):
#flag = 'depart-pro'
bodybuilder = [
"""

{i}
{department}
{colum}
{count}

""".format(
i=i,
colum= "{name}".format(name=idata.get('name', None)) if idata.get('name', None) else '',
**idata
)for i, idata in enumerate(data)
]
body = "/n".join(bodybuilder)
colum_dict= {
"depart-pro":"项目名称",
"depart-user":"发布人员",
}
return """





{colum}




{body}

序号 部门 上线次数

""".format(colum=colum_dict.get(flag, ''), body=body)
def _listify(v):
return list(v) if isinstance(v, (list, tuple)) else [v]
def send_email(msg):
#now = datetime.datetime.now()
#before = now-datetime.timedelta(days=1)
content_head = "
Hi,   all:
     该邮件为ship系统过去一周(上周五-本周五)上线次数统计, 提示: 无需回复/n
"
msg_mail = content_head + msg
def _format_addr(s):
name, addr = parseaddr(s)
return formataddr(( /
Header(name, 'utf-8').encode(), /
addr.encode('utf-8') if isinstance(addr, unicode) else addr))
from_addr = "**"
to_addr = "**"
to_cc_addr = "**"
smtp_server = "**"
msg = MIMEText(msg_mail, 'html', 'utf-8')
msg['From'] = _format_addr(u'** SA <%s>' % from_addr)
msg['To'] = to_addr
msg['Cc'] = to_cc_addr
msg['Subject'] = Header(u'【ship上线次数统计】', 'utf-8').encode()
server = smtplib.SMTP(smtp_server, 25)
server.set_debuglevel(1)
server.sendmail(from_addr,to_addr.split(',')+to_cc_addr.split(','), msg.as_string())
server.quit()
def main(*flags):
# flags = _listify(flag)
d = {
0:"1.各部门发布次数top10",
1:"2.各个项目发布次数top10",
2:"3.部署人员发布次数top10"
}
report_group = []
for iflag in flags:
idata = get_list(iflag)
ireport = render_data(idata,iflag)
if iflag == 'department':
report_group.append(d[0])
report_group.append(ireport)
elif iflag == 'depart-pro':
report_group.append(d[1])
report_group.append(ireport)
else:
report_group.append(d[2])
report_group.append(ireport)
#print ("report_group", report_group)
report_html = "
".join(report_group)
#with codecs.open("table_report_xxx.html", "w", 'utf8')as fd:
#fd.write(report_html)
#os.system("open table_report_xxx.html")
send_email(report_html)
if __name__ == '__main__':
main('department', 'depart-pro', 'depart-user')
#print get_data()

最新文章

123

最新摄影

微信扫一扫

第七城市微信公众平台