Commit 7a2aae3f authored by Benjamin Schwenker's avatar Benjamin Schwenker
Browse files

apply style fixes

parent efe7c617
......@@ -33,7 +33,6 @@ import datetime
import calendar
import logging
from pathlib import Path
import sys
import pickle
import os
import glob
......
......@@ -342,7 +342,7 @@ def make_summary(start, end, tasks, superkekb_archiver, data_folder, file_name,
success = True
# buffer data for later usage
logging.info('Success downloading {}'.format(pv) )
logging.info('Success downloading {}'.format(pv))
df.to_parquet(buffer_path)
# cut on timestamps
......
......@@ -562,7 +562,7 @@ SKB2_ACC_PVS = [
"MGHGL:HER:P", # Beam energy
"MGLGL:LER:P",
"CGHOBT:SLYTLE_1:X", # Offsets of strong sextupole magnets for chromatic coupling, HER ring, in mm
"CGHOBT:SLYTLE_1:X", # Offsets of strong sextupole magnets for chromatic coupling, HER ring, in mm
"CGHOBT:SLYTLE_2:X",
"CGHOBT:SLYTLE_1:Y",
"CGHOBT:SLYTLE_2:Y",
......@@ -571,7 +571,7 @@ SKB2_ACC_PVS = [
"CGHOBT:SLYTRE_1:Y",
"CGHOBT:SLYTRE_2:Y",
"CGLOBT:SLYTLP_1:X", # Offsets of strong sextupole magnets for chromatic coupling, LER ring, in mm
"CGLOBT:SLYTLP_1:X", # Offsets of strong sextupole magnets for chromatic coupling, LER ring, in mm
"CGLOBT:SLYTLP_2:X",
"CGLOBT:SLYTLP_1:Y",
"CGLOBT:SLYTLP_2:Y",
......
......@@ -324,6 +324,7 @@ def downsample_data(data, fraction=0.33):
return data
def downsample_data_to_npoints(data, npoints):
"""
Returns data after randomly removing (drop) fraction of rows.
......
......@@ -436,8 +436,7 @@ def sanity_check_and_repair_scaler(x, scaler_x, q1=10.0, q2=90.0):
for i in range(np.shape(x)[1]):
if ((scaler_x.scale_[i] == 1)
& (np.quantile(x[:, i], q1/100) == np.quantile(x[:, i], q2/100))
& (np.amax(x[:, i]) != np.amin(x[:, i]))
):
& (np.amax(x[:, i]) != np.amin(x[:, i]))):
scaler_x.center_[i] = np.quantile(x[:, i], q1/100)
scaler_x.scale_[i] = np.amax(x[:, i])-np.amin(x[:, i])
......@@ -521,7 +520,7 @@ def get_gated_injection_model(num_injection, name, hidden_units, hidden_layers):
# )(x1)
# Output gating 1: Output is multiplied by output of attention layer
#learned_gate = Dot(axes=1)([x1, attention])
# learned_gate = Dot(axes=1)([x1, attention])
inj_output = Dense(
hidden_units,
......@@ -1149,12 +1148,12 @@ def get_scaled_sensitivty(sensitivity, model, scalers_x, scaler_y):
sub_models = ['injection_her', 'injection_ler', 'her_beam_gas', 'her_touschek', 'ler_beam_gas', 'ler_touschek']
map_sub_models_to_decomp = {
'injection_her' : 'pred_inj_her',
'injection_ler' : 'pred_inj_ler',
'her_beam_gas' : 'pred_beam_gas_her',
'her_touschek' : 'pred_touschek_her',
'ler_beam_gas' : 'pred_beam_gas_ler',
'ler_touschek' : 'pred_touschek_ler',
'injection_her': 'pred_inj_her',
'injection_ler': 'pred_inj_ler',
'her_beam_gas': 'pred_beam_gas_her',
'her_touschek': 'pred_touschek_her',
'ler_beam_gas': 'pred_beam_gas_ler',
'ler_touschek': 'pred_touschek_ler',
}
index_map = {
......
......@@ -226,7 +226,7 @@ class elog():
retries += 1
if success:
break
if retries >=10:
if retries >= 10:
logging.warning('Elog timeout')
break
return result
......@@ -239,7 +239,7 @@ class elog():
while True:
try:
result = self._b2rc_session.post(self._b2rc_url + name,
data=data, files=files)
data=data, files=files)
result.raise_for_status()
success = True
except Exception as e:
......@@ -248,7 +248,7 @@ class elog():
retries += 1
if success:
break
if retries >=10:
if retries >= 10:
logging.warning('Elog timeout')
break
return result
......@@ -331,7 +331,7 @@ class elog():
retries += 1
if success:
break
if retries >=10:
if retries >= 10:
logging.warning('B2rc posting timeout')
break
return result.text
......@@ -347,7 +347,7 @@ class elog():
data['emoji'] = ':'.join(('', str(emoji).strip(':'), ''))
if self.avatar and self.avatar.startswith('http'):
data['avatar'] = self.avatar
result = self.api_post('/api/v1/chat.postMessage', data)
if not result:
logging.warning('Could not reach b2rc login')
......
......@@ -55,7 +55,8 @@ class ReportsToWebServer():
self.connectVPN.connect()
base_folder = "/home/benjamin/public_html/bgnet_reports/{}-{}-{}_{}".format(self.report_day.year, self.report_day.month, self.report_day.day, self.shift)
base_folder = "/home/benjamin/public_html/bgnet_reports/{}-{}-{}_{}".format(
self.report_day.year, self.report_day.month, self.report_day.day, self.shift)
os.system("sshpass -f {} ssh benjamin@130.87.225.93 'mkdir -p {}'".format(self.skbsrv_cred, base_folder))
for report in self.reports:
......@@ -64,4 +65,3 @@ class ReportsToWebServer():
self.connectVPN.disconnect()
return
......@@ -66,7 +66,20 @@ def two_axis_plot(obs, index, x, y, PV, PV_pos, reference_data_times, test_data_
return plt
def hitrate_two_axis_plot(obs, sub_model_pred_name, index, x, y, sub_model_pred_y, PV, PV_pos, reference_data_times, test_data_times, obs_unit="NA", PV_unit="NA", look_back=7):
def hitrate_two_axis_plot(
obs,
sub_model_pred_name,
index,
x,
y,
sub_model_pred_y,
PV,
PV_pos,
reference_data_times,
test_data_times,
obs_unit="NA",
PV_unit="NA",
look_back=7):
"""
Creates a plot with 2 y-axis, one for predicted observable and one for "PV".
Also highlights areas defined by "reference_data_times" and "test_data_times"
......@@ -114,6 +127,7 @@ def hitrate_two_axis_plot(obs, sub_model_pred_name, index, x, y, sub_model_pred_
fig.tight_layout()
return plt
def prediction_plot(obs, index, y, y_pred, reference_data_times, test_data_times, look_back=7, obs_unit="NA"):
"""
Creates a plot overlaying the prediction and observed data.
......
......@@ -64,7 +64,6 @@ import math
import os
import yaml
import sys
import glob
import subprocess
import logging
import traceback
......@@ -123,6 +122,7 @@ def get_report_day(year, month, day):
report_day = str_to_jst_datetime("{:04d}-{:02d}-{:02d} 00:00:00".format(year, month, day))
return report_day
def get_jobname(tag, channel, report_day, shift):
if "Daily report" in tag:
jobname = 'daily_report_{}_{}_{}_{}_{}'.format(shift, channel, report_day.year, report_day.month, report_day.day)
......@@ -131,6 +131,7 @@ def get_jobname(tag, channel, report_day, shift):
return jobname
def get_data_folder(archive):
"""Return output folder (directory) name for data"""
return os.path.expandvars(archive)
......@@ -177,10 +178,10 @@ def add_table_header(title):
def add_table_footer(title):
"""Returns latex code string for footer of table slide"""
ret = " \\end{tabular}\n"
ret +=" \\caption{MyTitle}\n"
ret +=" \\end{table}\n"
ret +=" \\end{tiny}\n"
ret +="\\end{frame}\n\n"
ret += " \\caption{MyTitle}\n"
ret += " \\end{table}\n"
ret += " \\end{tiny}\n"
ret += "\\end{frame}\n\n"
ret = ret.replace('MyTitle', get_latexed_feature(title))
return ret
......@@ -195,17 +196,16 @@ def add_table_content(feature_names, ncols=3):
def add_table(feature_names, title):
"""Returns latex code string for table slide"""
return add_table_header(title) + add_table_content(feature_names) + add_table_footer(title)
return add_table_header(title) + add_table_content(feature_names) + add_table_footer(title)
def compile_latex_inputs(features_dict):
"""Returns latex code string for compiling tables with input variable names"""
# Each table has 3 columns and can have up to 90 feature names
ncols = 3
# Each table has up to 90 feature names
nsplit = 90
# Latex code for compiling tables
# Latex code for compiling tables
latex_code = ""
for model, feature_names in features_dict.items():
......@@ -214,14 +214,12 @@ def compile_latex_inputs(features_dict):
ntables = math.ceil(len(feature_names)/nsplit)
for itable in range(ntables):
latexed_features = [name.replace("_", "\\_") for name in feature_names_split[itable]]
list_of_rows = ["&".join(latexed_features[i:i+ncols])+r"\\" for i in range(0, len(latexed_features), ncols)]
table_text = "\n".join(list_of_rows)
title = "List of input PVs for model {}, part {}".format(model, itable+1)
latex_code += add_table(feature_names_split[itable], title)
return latex_code
def compile_latex_report(
channel,
obs,
......@@ -250,7 +248,7 @@ def compile_latex_report(
with open(template_path, 'r') as file:
filedata = file.read()
# Count number of inputs, over all submodels
# Count number of inputs, over all submodels
ninputs = 0
for _, feature_names in features_dict.items():
ninputs += len(feature_names)
......@@ -273,8 +271,8 @@ def compile_latex_report(
filedata = filedata.replace('INPUTS_VAR', compile_latex_inputs(features_dict))
template_base_path = os.path.split(template_path)[0]
filedata = filedata.replace('GOE_LOGO_VAR', '{}'.format(os.path.join(template_base_path,"goe_Logo_2.jpg")))
filedata = filedata.replace('BELLE2_LOGO_VAR', '{}'.format(os.path.join(template_base_path,"belle2-logo.png")))
filedata = filedata.replace('GOE_LOGO_VAR', '{}'.format(os.path.join(template_base_path, "goe_Logo_2.jpg")))
filedata = filedata.replace('BELLE2_LOGO_VAR', '{}'.format(os.path.join(template_base_path, "belle2-logo.png")))
# Write the file out again
texfile = os.path.join(tmpdirname, os.path.basename(template_path))
......@@ -530,7 +528,6 @@ def report_on_hitrate(
logging.error(traceback.format_exc())
return
if x[0].size == 0:
logging.info('Error: No data for training found. Quit reporting on {}'.format(obs))
return
......@@ -723,7 +720,7 @@ def report_on_hitrate(
PV_name = ranking[i]
PV_pos = feature_names.index(PV_name)
if not "GATE" in PV_name and count_two_axis_plots<=3:
if "GATE" not in PV_name and count_two_axis_plots <= 3:
plot = hitrate_two_axis_plot(
obs,
sub_model_pred_name=sub_model_pred_name,
......@@ -739,9 +736,18 @@ def report_on_hitrate(
PV_unit="NA",
look_back=look_back,)
plot.title("Attribution rank: {} Mean attribution value: {:.3f} {}".format(i, mean_abs[PV_pos], obs_unit), y=1.2)
plot.title(
"Attribution rank: {} Mean attribution value: {:.3f} {}".format(
i, mean_abs[PV_pos], obs_unit), y=1.2)
plot.tight_layout()
plot.savefig(os.path.join(save_path, "rank{}_{}_{}.pdf".format(count_two_axis_plots, channel, sub_model_name)), bbox_inches='tight')
plot.savefig(
os.path.join(
save_path,
"rank{}_{}_{}.pdf".format(
count_two_axis_plots,
channel,
sub_model_name)),
bbox_inches='tight')
plot.close('all')
count_two_axis_plots += 1
......@@ -762,9 +768,12 @@ def report_on_hitrate(
train_times=train_times,
attr_ref_times=ref_times[0],
attr_test_times=test_times[0],
features_dict={sub_model: hitrate.get_interpretable_feature_names(config, name=sub_model) for sub_model in hitrate.sub_models},
features_dict={
sub_model: hitrate.get_interpretable_feature_names(
config,
name=sub_model) for sub_model in hitrate.sub_models},
report_path=report_path,
)
)
return
......@@ -782,7 +791,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
Path(report_path).mkdir(parents=True, exist_ok=True)
# Report on hitrate
config = get_config(os.path.join(config_path,'training_default.yaml'))
config = get_config(os.path.join(config_path, 'training_default.yaml'))
channel = "VXD_Rad_BP_BW_325_DoseRate"
report_on_hitrate(
......@@ -792,7 +801,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
obs_unit="mRad/s (uncalibrated)",
config=config,
feature_names=[],
template_path=os.path.join(config_path,'templates/daily_report_hitrate_template.tex'),
template_path=os.path.join(config_path, 'templates/daily_report_hitrate_template.tex'),
year=year,
month=month,
day=day,
......@@ -816,7 +825,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
channels.append(channel)
# Report on hitrate
config = get_config(os.path.join(config_path,'training_default.yaml'))
config = get_config(os.path.join(config_path, 'training_default.yaml'))
channel = "CDC_CUR_AVERAGE"
report_on_hitrate(
......@@ -826,7 +835,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
obs_unit="mA",
config=config,
feature_names=[],
template_path=os.path.join(config_path,'templates/daily_report_hitrate_template.tex'),
template_path=os.path.join(config_path, 'templates/daily_report_hitrate_template.tex'),
year=year,
month=month,
day=day,
......@@ -850,7 +859,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
channels.append(channel)
# Report on channel lumi
config = get_config(os.path.join(config_path,'training_lumi.yaml'))
config = get_config(os.path.join(config_path, 'training_lumi.yaml'))
channel = 'lumi'
report_on_channel(
......@@ -860,7 +869,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
obs_unit=get_unit(lumi.OBS),
config=config,
feature_names=config['preparation']['lumi_pvs'],
template_path=os.path.join(config_path,'templates/daily_report_lumi_template.tex'),
template_path=os.path.join(config_path, 'templates/daily_report_lumi_template.tex'),
year=year,
month=month,
day=day,
......@@ -884,7 +893,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
channels.append(channel)
# Report on channel injection duration HER
config = get_config(os.path.join(config_path,'training_inj_duration.yaml'))
config = get_config(os.path.join(config_path, 'training_inj_duration.yaml'))
channel = 'HER_INJ'
report_on_channel(
......@@ -894,7 +903,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
obs_unit=get_unit(injection_duration.OBS_HER),
config=config,
feature_names=config['preparation']['her_inj_pvs'],
template_path=os.path.join(config_path,'templates/daily_report_inj_her_template.tex'),
template_path=os.path.join(config_path, 'templates/daily_report_inj_her_template.tex'),
year=year,
month=month,
day=day,
......@@ -918,7 +927,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
channels.append(channel)
# Report on channel injection duration LER
config = get_config(os.path.join(config_path,'training_inj_duration.yaml'))
config = get_config(os.path.join(config_path, 'training_inj_duration.yaml'))
channel = 'LER_INJ'
report_on_channel(
......@@ -928,7 +937,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
obs_unit=get_unit(injection_duration.OBS_LER),
config=config,
feature_names=config['preparation']['ler_inj_pvs'],
template_path=os.path.join(config_path,'templates/daily_report_inj_ler_template.tex'),
template_path=os.path.join(config_path, 'templates/daily_report_inj_ler_template.tex'),
year=year,
month=month,
day=day,
......@@ -952,7 +961,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
channels.append(channel)
# Report on channel beam life HER
config = get_config(os.path.join(config_path,'training_beam_life.yaml'))
config = get_config(os.path.join(config_path, 'training_beam_life.yaml'))
channel = 'HER_LIFE'
report_on_channel(
......@@ -962,7 +971,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
obs_unit=get_unit(beam_life.OBS_HER),
config=config,
feature_names=config['preparation']['her_life_pvs'],
template_path=os.path.join(config_path,'templates/daily_report_life_her_template.tex'),
template_path=os.path.join(config_path, 'templates/daily_report_life_her_template.tex'),
year=year,
month=month,
day=day,
......@@ -986,7 +995,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
channels.append(channel)
# Report on channel beam life LER
config = get_config(os.path.join(config_path,'training_beam_life.yaml'))
config = get_config(os.path.join(config_path, 'training_beam_life.yaml'))
channel = 'LER_LIFE'
report_on_channel(
......@@ -996,7 +1005,7 @@ def make_reports(year, month, day, data_path, tag, shift, look_back, ref_times,
obs_unit=get_unit(beam_life.OBS_LER),
config=config,
feature_names=config['preparation']['ler_life_pvs'],
template_path=os.path.join(config_path,'templates/daily_report_life_ler_template.tex'),
template_path=os.path.join(config_path, 'templates/daily_report_life_ler_template.tex'),
year=year,
month=month,
day=day,
......@@ -1124,7 +1133,7 @@ def main():
file_name = get_jobname(shift=args.shift, channel=channel, report_day=start, tag=args.tag) + '.pdf'
file_path = os.path.join(args.report_path, file_name)
if Path(file_path).is_file():
attachments.append( file_path )
attachments.append(file_path)
# Should guarantee fixed order of attachments in eLog entry
attachments = sorted(attachments)
......@@ -1157,5 +1166,6 @@ def main():
else:
logging.info("Created reports: \n" + ", ".join(attachments))
if __name__ == '__main__':
main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment