-
Notifications
You must be signed in to change notification settings - Fork 3
/
parseWMStatsFailures.py
65 lines (53 loc) · 3.37 KB
/
parseWMStatsFailures.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#!/usr/bin/env python
"""
Fetch data from wmstats - for a specific workflow - and print a few tasks and the number
of job failures for CERN sites (read known sites that stage to T2_CH_CERN).
"""
import sys
import os
import json
import httplib
from pprint import pformat
def getWMStatsData(workflow):
url = 'cmsweb.cern.ch'
headers = {"Accept": "application/json", "Content-type": "application/json"}
conn = httplib.HTTPSConnection(url, cert_file = os.getenv('X509_USER_PROXY'), key_file = os.getenv('X509_USER_PROXY'))
urn = "/wmstatsserver/data/request/%s" % workflow
conn.request("GET", urn, headers=headers)
r2=conn.getresponse()
request = json.loads(r2.read())["result"][0]
return request
def totalCERNFailures(data):
summary = {k: v.get('failure') for k, v in data.items() if "CERN" in k}
success = [v.get('success', 0) for k, v in data.items() if "CERN" in k]
success = sum(success)
fail = 0
for site, values in summary.items():
if values is None:
continue
for failtype, num in values.items():
fail += num
return "%s/%s" % (success, fail)
def main():
data = getWMStatsData("pdmvserv_task_HIG-RunIIFall17wmLHEGS-01116__v1_T_180418_043017_9486")
data = data['pdmvserv_task_HIG-RunIIFall17wmLHEGS-01116__v1_T_180418_043017_9486']['AgentJobInfo']
for agent in data.keys():
print("\nStats for agent: %s" % agent)
print(" Skipped files: %s" % pformat(data[agent]['skipped']))
print(" Overall agent status:\t\t %s" % data[agent]['status'])
fail = totalCERNFailures(data[agent]['tasks']['/pdmvserv_task_HIG-RunIIFall17wmLHEGS-01116__v1_T_180418_043017_9486/HIG-RunIIFall17wmLHEGS-01116_0']['sites'])
print(" Task1 success/failures at CERN: %s" % fail)
fail = totalCERNFailures(data[agent]['tasks']['/pdmvserv_task_HIG-RunIIFall17wmLHEGS-01116__v1_T_180418_043017_9486/HIG-RunIIFall17wmLHEGS-01116_0/HIG-RunIIFall17wmLHEGS-01116_0CleanupUnmergedLHEoutput']['sites'])
print(" Task1 Cleanup LHE success/failures at CERN: %s" % fail)
fail = totalCERNFailures(data[agent]['tasks']['/pdmvserv_task_HIG-RunIIFall17wmLHEGS-01116__v1_T_180418_043017_9486/HIG-RunIIFall17wmLHEGS-01116_0/HIG-RunIIFall17wmLHEGS-01116_0CleanupUnmergedRAWSIMoutput']['sites'])
print(" Task1 Cleanup RAWSIM success/failures at CERN: %s" % fail)
fail = totalCERNFailures(data[agent]['tasks']['/pdmvserv_task_HIG-RunIIFall17wmLHEGS-01116__v1_T_180418_043017_9486/HIG-RunIIFall17wmLHEGS-01116_0/HIG-RunIIFall17DRPremix-00782_0']['sites'])
print(" Task2 success/failures at CERN: %s" % fail)
fail = totalCERNFailures(data[agent]['tasks']['/pdmvserv_task_HIG-RunIIFall17wmLHEGS-01116__v1_T_180418_043017_9486/HIG-RunIIFall17wmLHEGS-01116_0/HIG-RunIIFall17DRPremix-00782_0/HIG-RunIIFall17DRPremix-00782_0CleanupUnmergedPREMIXRAWoutput']['sites'])
print(" Task2 Cleanup PREMIXRAW success/failures at CERN: %s" % fail)
fail = totalCERNFailures(data[agent]['tasks']['/pdmvserv_task_HIG-RunIIFall17wmLHEGS-01116__v1_T_180418_043017_9486/HIG-RunIIFall17wmLHEGS-01116_0/HIG-RunIIFall17DRPremix-00782_0/HIG-RunIIFall17DRPremix-00782_1']['sites'])
print(" Task3 success/failures at CERN: %s" % fail)
print "Done!"
sys.exit(0)
if __name__ == '__main__':
sys.exit(main())