dsmakarov's picture
fixed averaging for reversed frq sort case
3596592
Raw History Blame Contribute Delete
10.5 kB
from shiny.types import FileInfo, NavSetArg
from shiny import *
import numpy as np
from pathlib import Path
import json
import zipfile
import io
import math
from matplotlib import pyplot as plt
import pandas as pd
import time
from utils_process import multipass_average, multipass_average2, rotate_data, frq_recalc
app_ui = ui.page_fluid(
ui.panel_title("RAD data processing"),
ui.layout_sidebar(ui.panel_sidebar(ui.input_file("aux_data", "Load recordings info", accept=["*.*"], multiple=False)),
ui.panel_main(ui.output_ui("aux_data_show"))
),
ui.row(ui.column(4, ui.input_file("input_files", "Load your spectra", accept=["*.*"], multiple=True)),
ui.column(4, ui.input_text("text_comment", "Commented lines", value="//"))
),
ui.row(ui.column(3, ui.input_numeric("pv_rows", "Rows", 1, min=1, max=10, step=1)),
ui.column(3, ui.input_numeric("pv_cols", "Cols", 6, min=1, max=10, step=1))
),
ui.input_action_button("b_preview", "Preview raw data"),
ui.output_plot("preview_spectrum"),
ui.row(ui.column(4, ui.input_numeric("n_harm", "N harmonic", 20, min=1, max=50, step=1)),
ui.column(4, ui.input_numeric("ifrq", "IF value, MHz", 350.)),
),
ui.input_numeric("cell_length", 'Cell length, cm', 10.),
ui.input_switch("s_normalize", "Normalize data"),
ui.input_action_button("b_view_processed", "Preview processed data"),
ui.output_plot("preview_processed"),
ui.row(ui.column(4, ui.input_text("prefix_out", "Output file prefix", value="")),
ui.column(4, ui.input_text("extens_out", "Output file extension", value="txt"))),
ui.row(ui.column(4, ui.input_switch("is_foreign", "Foreign broadening")),
ui.column(4, ui.input_numeric("p_self", "Self pressure", 0.000))),
ui.row(ui.column(4, ui.download_button("save_spectra", "Save spectra")),
ui.column(4, ui.download_button("save_aux", "Save aux data")))
)
def server(input, output, session):
raw_data: reactive.Value[list] = reactive.Value([])
names: reactive.Value[list] = reactive.Value([])
f_aux = reactive.Value(False) # flag that aux info loaded
nrecs = reactive.Value(0)
ready_data: reactive.Value[list] = reactive.Value([])
aux_df: pd.DataFrame = reactive.Value()
aux_df_out: pd.DataFrame = reactive.Value()
# transfer data from loaded files into globally visible variables
@reactive.Effect
@reactive.event(input.input_files)
def _():
tmp_recs = []
tmp_names = []
f: list[FileInfo] = input.input_files()
# f has fields: name, size, type, datapath
for ifil in range(len(f)):
tmp_names.append(f[ifil]["name"])
cur_data = np.loadtxt(f[ifil]["datapath"], comments=input.text_comment())
tmp_recs.append(cur_data)
raw_data.set(tmp_recs)
nrecs.set(len(f))
names.set(tmp_names)
@output
@render.ui
def aux_data_show():
if input.aux_data() is None:
return 'No recording conditions data loaded'
aux_file: list[FileInfo] = input.aux_data() # read data on the aux file (name, size, type, datapath)
aux_names = ['File', 'P', 'T', 'dev'] # column names of the aux data
aux_out = pd.read_csv(aux_file[0]["datapath"], names=aux_names, delim_whitespace=True, index_col=0) # make a dataframe from the aux data
aux_df.set(aux_out)
return ui.HTML(aux_out.to_html(classes="table table-striped"))
@output
@render.plot(alt="Preview of the loaded spectra")
@reactive.event(input.b_preview)
def preview_spectrum():
if input.input_files() is None:
fig, ax = plt.subplots()
ax = plt.text(0.4, 0.4, "Please upload your spectra first")
return fig
fig, axs = plt.subplots(nrows=int(input.pv_rows()), ncols=int(input.pv_cols()),
sharex=False, sharey=True,
figsize=(input.pv_cols()*2.5, input.pv_rows()*2.5), squeeze=False)
ax = axs.flat
for ifil in range(nrecs.get()):
tmp_rec = raw_data.get()[ifil]
tmp_synth = tmp_rec[:, 0]
if ifil==0:
tmp_ctr = 0.5 * (min(tmp_synth) + max(tmp_synth))
ax[ifil].text(0.1, 0.95,
f'{tmp_ctr*1000:.3f}'+ ' MHz',
transform=ax[ifil].transAxes)
synth_uniq, synth_counts = np.unique(tmp_synth, return_counts=True)
tmp_pps = synth_uniq.shape[0]
tmp_nscan = synth_counts[0]
tmp_ch1 = tmp_rec[:, 1]
tmp_ch2 = tmp_rec[:, 2]
ax[ifil].errorbar((tmp_synth - tmp_ctr)*1000, tmp_ch1, fmt='ro', ms=0.5)
ax[ifil].errorbar((tmp_synth - tmp_ctr)*1000, tmp_ch2, fmt='bo', ms=0.5)
ax[ifil].text(0.1, 0.9, 'Pps: '+str(tmp_pps) if ifil==0 else str(tmp_pps), transform=ax[ifil].transAxes)
ax[ifil].text(0.1, 0.85, 'Scans: '+str(tmp_nscan), transform=ax[ifil].transAxes)
if min(synth_uniq)==0.:
ax[ifil].text(0.1, 0.8, 'UNFINISHED SCANS', transform=ax[ifil].transAxes)
ax[ifil].set_xlabel('Frequency detuning, MHz')
return fig
@output
@render.plot(alt="Preview of the processed spectra")
@reactive.event(input.b_view_processed)
def preview_processed():
if input.input_files() is None:
fig, ax = plt.subplots()
ax = plt.text(0.4, 0.4, "Please upload your spectra first")
return fig
fig, axs = plt.subplots(nrows=int(input.pv_rows()), ncols=int(input.pv_cols()),
sharex=False, sharey=True,
figsize=(input.pv_cols()*2.5, input.pv_rows()*2.5), squeeze=False)
ax = axs.flat
tmp_processed = []
for ifil in range(nrecs.get()):
tmp_rec = raw_data.get()[ifil]
tmp_synth = tmp_rec[:, 0] # synth frequencies from RAD data
synth_uniq, synth_counts = np.unique(tmp_synth, return_counts=True) # use numpy functions to retrieve necessary metadata
tmp_pps = synth_uniq.shape[0] # points per scan from RAD data
tmp_nscan = synth_counts[0] # number of scans from RAD data
tmp_ch1 = tmp_rec[:, 1] # signal channel 1
tmp_ch2 = tmp_rec[:, 2] # signal channel 2
frq_ready = frq_recalc(synth_uniq*1000., input.n_harm(), input.ifrq())
tmp_ctr = 0.5 * (min(frq_ready) + max(frq_ready))
# ch1_ready = multipass_average(tmp_ch1, tmp_pps)
# ch2_ready = multipass_average(tmp_ch2, tmp_pps)
ch1_ready, ch2_ready = multipass_average2(tmp_rec, tmp_pps)
if input.s_normalize():
ch2_ready[:] = ch2_ready[:]/max(ch1_ready)
ch1_ready[:] = ch1_ready[:]/max(ch1_ready)
tmp_processed.append(np.stack((frq_ready, ch1_ready), axis=1))
ax[ifil].errorbar((frq_ready - tmp_ctr), ch1_ready, fmt='r-', lw=1.)
ax[ifil].errorbar((frq_ready - tmp_ctr), ch2_ready, fmt='b-', lw=1.)
# ax[ifil].text(0.1, 0.9, 'Points per scan: '+str(tmp_pps), transform=ax[ifil].transAxes)
# ax[ifil].text(0.1, 0.85, 'Scans: '+str(tmp_nscan), transform=ax[ifil].transAxes)
if ifil==0:
tmp_ctr = 0.5 * (min(frq_ready) + max(frq_ready))
ax[ifil].text(0.1, 0.95,
f'{tmp_ctr:.3f}'+ ' MHz',
transform=ax[ifil].transAxes)
# if min(synth_uniq)==0.:
# ax[ifil].text(0.1, 0.8, 'UNFINISHED SCANS', transform=ax[ifil].transAxes)
ax[ifil].set_xlabel('Frequency detuning, MHz')
ready_data.set(tmp_processed)
return fig
@render.download(filename='spectra.zip')
def save_spectra():
with io.BytesIO() as buf:
# this creates in-memory buffer where zip archive is stored
# without storing file on the disk
testzip = zipfile.ZipFile(buf, 'w') # create a ZipFile object, instead of filename use buffer object name
tmp_names = names.get() # filenames specified in the list of experimental conditions
tmp_data = ready_data.get() # processed recordings
for i in range(len(tmp_names)):
current_name = input.prefix_out()+tmp_names[i][:-3]+input.extens_out() # add prefix and change extension of the filename
current_df = pd.DataFrame(tmp_data[i]) # load recording into dataframe (i don't remember why so, but this works best anyway)
testzip.writestr(current_name, current_df.to_string(header=False, index=False)) # write recording in a file within the archive
testzip.close()
yield buf.getvalue() # return the buffer content as file to download
@render.download(filename='aux_data.txt')
def save_aux():
tmp_names = names.get()
tmp_names_out = [input.prefix_out()+tmp_name[:-3]+input.extens_out() for tmp_name in tmp_names] # add prefix and change extension of the filename
aux_data = np.zeros((len(tmp_names), 6))
tmp_aux = aux_df.get()
p_total = tmp_aux['P'].values
exp_pself = np.zeros_like(p_total)
exp_pfor = np.zeros_like(p_total)
if input.is_foreign():
if input.p_self() == 0.0:
exp_pself[:] = p_total[0]
else:
exp_pself[:] = input.p_self()
exp_pfor[:] = p_total[:] - exp_pself[:]
else:
exp_pself[:] = p_total[:]
aux_data[:, 0] = exp_pself
aux_data[:, 1] = exp_pfor
aux_data[:, 2] = tmp_aux['T'].values
aux_data[:, 3] = tmp_aux['dev'].values
aux_data[:, 4] = 1
aux_data[:, 5] = input.cell_length()
for ifil in range(len(tmp_names_out)):
yield ' '.join([tmp_names_out[ifil].ljust(20)]
+ [('%7.3f' % elem).ljust(7) for elem in aux_data[ifil,0:4]]
+ ['%1i' % aux_data[ifil, 4],'%6.3f' % aux_data[ifil, 5]]
+ ['\n'])
app = App(app_ui, server)