import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
plt.rcParams['figure.constrained_layout.use'] = True

import dask
print(f'dask: {dask.__version__}')
import dask.array
dask.config.set({'array.chunk-size': '512MiB'})

from psutil import virtual_memory

mem = virtual_memory()
print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb')  # total physical memory available

import logging
logging.basicConfig(filename='example.log', level=logging.DEBUG)

import toolbox_scs as tb
print(tb.__file__)
import toolbox_scs.routines.boz as boz

from extra_data.read_machinery import find_proposal

import os

dask: 2.11.0
Physical memory: 504 Gb
/home/lleguy/notebooks/ToolBox/src/toolbox_scs/__init__.py

%load_ext autoreload

%autoreload 2

Create parameters object

proposal = 2937
darkrunNB = 671
runNB = 670
moduleNB = 15
gain = 3
sat_level = 500
rois_th = 4

root = find_proposal(f'p{proposal:06d}')
path = root + f'/usr/processed_runs/'
params = boz.parameters.load(path + 'r0614/parameters_p2937_d615_r614.json')

rpath = path + f'r{runNB:04d}/'
prefix = prefix = f'p{proposal}-r{runNB}-d{darkrunNB}-BOZ-Ib'
os.makedirs(rpath, exist_ok=True)

Dark run inspection

The aim is to check dark level and extract bad pixel map.

arr_dark, tid_dark = boz.load_dssc_module(proposal, darkrunNB, moduleNB)
arr_dark = arr_dark.rechunk(('auto', -1, -1, -1))

f = boz.inspect_dark(arr_dark)
f.suptitle(f'p:{proposal} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_dark.png'
f.savefig(fname, dpi=300)

Veto pattern check

Check potential veto pattern issue

arr, tid = boz.load_dssc_module(proposal, runNB, moduleNB)
arr = arr.rechunk(('auto', -1, -1, -1))

dark = boz.average_module(arr_dark).compute()
data = boz.average_module(arr, dark=dark).compute()
pp = data.mean(axis=(1,2)) # pulseId resolved mean
dataM = data.mean(axis=0) # mean over pulseId

plt.figure()
plt.plot(pp)
plt.xlabel('pulseId')
plt.ylabel('dark corrected module mean')
plt.title(f'p:{proposal} r:{runNB} d:{darkrunNB}')
plt.savefig(rpath+prefix+'-veto_inspect.png', dpi=300)

"""
# Thresholding out bad veto pulse
if False:
    params.arr = params.arr[:, pp > 2, :, :]
    params.arr_dark = params.arr_dark[:, pp > 2, :, :]
    dark = boz.average_module(params.arr_dark).compute()
    data = boz.average_module(params.arr, dark=dark).compute()
    dataM = data.mean(axis=0) # mean over pulseId
"""

'\n# Thresholding out bad veto pulse\nif False:\n    params.arr = params.arr[:, pp > 2, :, :]\n    params.arr_dark = params.arr_dark[:, pp > 2, :, :]\n    dark = boz.average_module(params.arr_dark).compute()\n    data = boz.average_module(params.arr, dark=dark).compute()\n    dataM = data.mean(axis=0) # mean over pulseId\n'

Histogram

h, f = boz.inspect_histogram(arr,
                             arr_dark,
                             mask=params.get_mask() #, extra_lines=True
                         )
f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
f.savefig(rpath+prefix+'-histogram.png', dpi=300)

Check ROIs

Let’s check the ROIs used in the part I on a run later

it’s a bit off, also we can see from the blur that the photon energy was varied in this run.

rois_th = 4
rois = boz.find_rois(dataM, rois_th)

#rois = params.rois

f = boz.inspect_rois(dataM, rois, rois_th)

f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_rois.png'
f.savefig(fname, dpi=300)

Flat field extraction

The first step is to compute a good average image, this mean remove saturated shots and ignoring bad pixels

params.sat_level = sat_level
res = boz.average_module(arr, dark=dark,
                         ret='mean', mask=params.get_mask(), sat_roi=params.rois['sat'],
                         sat_level=params.sat_level)
avg = res.compute().mean(axis=0)

The second step is from that good average image to fit the plane field on n/0 and p/0 rois. We have to make sure that the rois have same width.

f = boz.inspect_plane_fitting(avg, rois)

f.savefig(rpath+prefix+'-inspect-noflatfield.png', dpi=300)

ff = boz.compute_flat_field_correction(rois, params)

f = boz.inspect_plane_fitting(avg/ff, rois)

f.savefig(rpath+prefix+'-inspect-withflatfield.png', dpi=300)

Non-linearities correction extraction

params.arr = arr
params.tid = tid
params.arr_dark = arr_dark

f = boz.inspect_correction(params, gain=params.gain)

mu: 0.5542050522036303, s: 0.0048626186034544635, snr: 113.97255211624375
weighted mu: 0.5545223884175955, s: 0.00439720741390005, snr: 126.10785351282044
diff mu: 0.5542050522036303, s: 0.004554834748790502, snr: 121.67401953513128
mu: 0.6018044784893213, s: 0.00509352230389592, snr: 118.15094596307443
weighted mu: 0.6021455860405776, s: 0.004582072582774097, snr: 131.41336702179086
diff mu: 0.6018044784893213, s: 0.004816496211705332, snr: 124.94652794011976
mu: 0.9209777163834091, s: 0.011727583302310964, snr: 78.53090382243774
weighted mu: 0.9209108250113904, s: 0.010867219280890697, snr: 84.74208545977835
diff mu: 0.9209777163834091, s: 0.010402507377172816, snr: 88.53420459036595
mu: 0.9875361525315763, s: 0.006361084864086151, snr: 155.24649861332233
weighted mu: 0.9876406078328054, s: 0.00525879919677722, snr: 187.8072485517353
diff mu: 0.9875361525315763, s: 0.006363818456805551, snr: 155.17981212608166
mu: 0.9949634261232898, s: 0.006202287767288881, snr: 160.41877827255414
weighted mu: 0.9956087812174397, s: 0.005361139651634466, snr: 185.70842132677996
diff mu: 0.9949634261232898, s: 0.0061959766161234515, snr: 160.58217901180436
mu: 0.9925391344667838, s: 0.0032376964134997726, snr: 306.5571961374548
weighted mu: 0.9919966822963425, s: 0.0028433364944561493, snr: 348.88472899022236
diff mu: 0.9925391344667838, s: 0.003109681420985185, snr: 319.1771117673962
mu: 0.9954331700375585, s: 0.0026042566774445186, snr: 382.2331257356507
weighted mu: 0.9953166337696191, s: 0.0022636639774922906, snr: 439.6927475394297
diff mu: 0.9954331700375585, s: 0.002585970212809487, snr: 384.936054215445
mu: 1.0018859826944055, s: 0.0021875817956110146, snr: 457.98789544898744
weighted mu: 1.0020339348661713, s: 0.0018944436735191359, snr: 528.9330840883666
diff mu: 1.0018859826944055, s: 0.002167571671170781, snr: 462.215850123771
mu: 0.9935623713612618, s: 0.002842988778942791, snr: 349.4781191963526
weighted mu: 0.9932963337240177, s: 0.00255672605023837, snr: 388.50323194829974
diff mu: 0.9935623713612618, s: 0.002722965705771748, snr: 364.8824402214293

f.savefig(rpath+prefix+'-inspect-correction.png', dpi=300)