from shiny.types import FileInfo, NavSetArg from shiny import * import numpy as np from pathlib import Path import json import zipfile import io import math from matplotlib import pyplot as plt import pandas as pd import time from utils_process import multipass_average, multipass_average2, rotate_data, frq_recalc app_ui = ui.page_fluid( ui.panel_title("RAD data processing"), ui.layout_sidebar(ui.panel_sidebar(ui.input_file("aux_data", "Load recordings info", accept=["*.*"], multiple=False)), ui.panel_main(ui.output_ui("aux_data_show")) ), ui.row(ui.column(4, ui.input_file("input_files", "Load your spectra", accept=["*.*"], multiple=True)), ui.column(4, ui.input_text("text_comment", "Commented lines", value="//")) ), ui.row(ui.column(3, ui.input_numeric("pv_rows", "Rows", 1, min=1, max=10, step=1)), ui.column(3, ui.input_numeric("pv_cols", "Cols", 6, min=1, max=10, step=1)) ), ui.input_action_button("b_preview", "Preview raw data"), ui.output_plot("preview_spectrum"), ui.row(ui.column(4, ui.input_numeric("n_harm", "N harmonic", 20, min=1, max=50, step=1)), ui.column(4, ui.input_numeric("ifrq", "IF value, MHz", 350.)), ), ui.input_numeric("cell_length", 'Cell length, cm', 10.), ui.input_switch("s_normalize", "Normalize data"), ui.input_action_button("b_view_processed", "Preview processed data"), ui.output_plot("preview_processed"), ui.row(ui.column(4, ui.input_text("prefix_out", "Output file prefix", value="")), ui.column(4, ui.input_text("extens_out", "Output file extension", value="txt"))), ui.row(ui.column(4, ui.input_switch("is_foreign", "Foreign broadening")), ui.column(4, ui.input_numeric("p_self", "Self pressure", 0.000))), ui.row(ui.column(4, ui.download_button("save_spectra", "Save spectra")), ui.column(4, ui.download_button("save_aux", "Save aux data"))) ) def server(input, output, session): raw_data: reactive.Value[list] = reactive.Value([]) names: reactive.Value[list] = reactive.Value([]) f_aux = reactive.Value(False) # flag that aux info loaded nrecs = reactive.Value(0) ready_data: reactive.Value[list] = reactive.Value([]) aux_df: pd.DataFrame = reactive.Value() aux_df_out: pd.DataFrame = reactive.Value() # transfer data from loaded files into globally visible variables @reactive.Effect @reactive.event(input.input_files) def _(): tmp_recs = [] tmp_names = [] f: list[FileInfo] = input.input_files() # f has fields: name, size, type, datapath for ifil in range(len(f)): tmp_names.append(f[ifil]["name"]) cur_data = np.loadtxt(f[ifil]["datapath"], comments=input.text_comment()) tmp_recs.append(cur_data) raw_data.set(tmp_recs) nrecs.set(len(f)) names.set(tmp_names) @output @render.ui def aux_data_show(): if input.aux_data() is None: return 'No recording conditions data loaded' aux_file: list[FileInfo] = input.aux_data() # read data on the aux file (name, size, type, datapath) aux_names = ['File', 'P', 'T', 'dev'] # column names of the aux data aux_out = pd.read_csv(aux_file[0]["datapath"], names=aux_names, delim_whitespace=True, index_col=0) # make a dataframe from the aux data aux_df.set(aux_out) return ui.HTML(aux_out.to_html(classes="table table-striped")) @output @render.plot(alt="Preview of the loaded spectra") @reactive.event(input.b_preview) def preview_spectrum(): if input.input_files() is None: fig, ax = plt.subplots() ax = plt.text(0.4, 0.4, "Please upload your spectra first") return fig fig, axs = plt.subplots(nrows=int(input.pv_rows()), ncols=int(input.pv_cols()), sharex=False, sharey=True, figsize=(input.pv_cols()*2.5, input.pv_rows()*2.5), squeeze=False) ax = axs.flat for ifil in range(nrecs.get()): tmp_rec = raw_data.get()[ifil] tmp_synth = tmp_rec[:, 0] if ifil==0: tmp_ctr = 0.5 * (min(tmp_synth) + max(tmp_synth)) ax[ifil].text(0.1, 0.95, f'{tmp_ctr*1000:.3f}'+ ' MHz', transform=ax[ifil].transAxes) synth_uniq, synth_counts = np.unique(tmp_synth, return_counts=True) tmp_pps = synth_uniq.shape[0] tmp_nscan = synth_counts[0] tmp_ch1 = tmp_rec[:, 1] tmp_ch2 = tmp_rec[:, 2] ax[ifil].errorbar((tmp_synth - tmp_ctr)*1000, tmp_ch1, fmt='ro', ms=0.5) ax[ifil].errorbar((tmp_synth - tmp_ctr)*1000, tmp_ch2, fmt='bo', ms=0.5) ax[ifil].text(0.1, 0.9, 'Pps: '+str(tmp_pps) if ifil==0 else str(tmp_pps), transform=ax[ifil].transAxes) ax[ifil].text(0.1, 0.85, 'Scans: '+str(tmp_nscan), transform=ax[ifil].transAxes) if min(synth_uniq)==0.: ax[ifil].text(0.1, 0.8, 'UNFINISHED SCANS', transform=ax[ifil].transAxes) ax[ifil].set_xlabel('Frequency detuning, MHz') return fig @output @render.plot(alt="Preview of the processed spectra") @reactive.event(input.b_view_processed) def preview_processed(): if input.input_files() is None: fig, ax = plt.subplots() ax = plt.text(0.4, 0.4, "Please upload your spectra first") return fig fig, axs = plt.subplots(nrows=int(input.pv_rows()), ncols=int(input.pv_cols()), sharex=False, sharey=True, figsize=(input.pv_cols()*2.5, input.pv_rows()*2.5), squeeze=False) ax = axs.flat tmp_processed = [] for ifil in range(nrecs.get()): tmp_rec = raw_data.get()[ifil] tmp_synth = tmp_rec[:, 0] # synth frequencies from RAD data synth_uniq, synth_counts = np.unique(tmp_synth, return_counts=True) # use numpy functions to retrieve necessary metadata tmp_pps = synth_uniq.shape[0] # points per scan from RAD data tmp_nscan = synth_counts[0] # number of scans from RAD data tmp_ch1 = tmp_rec[:, 1] # signal channel 1 tmp_ch2 = tmp_rec[:, 2] # signal channel 2 frq_ready = frq_recalc(synth_uniq*1000., input.n_harm(), input.ifrq()) tmp_ctr = 0.5 * (min(frq_ready) + max(frq_ready)) # ch1_ready = multipass_average(tmp_ch1, tmp_pps) # ch2_ready = multipass_average(tmp_ch2, tmp_pps) ch1_ready, ch2_ready = multipass_average2(tmp_rec, tmp_pps) if input.s_normalize(): ch2_ready[:] = ch2_ready[:]/max(ch1_ready) ch1_ready[:] = ch1_ready[:]/max(ch1_ready) tmp_processed.append(np.stack((frq_ready, ch1_ready), axis=1)) ax[ifil].errorbar((frq_ready - tmp_ctr), ch1_ready, fmt='r-', lw=1.) ax[ifil].errorbar((frq_ready - tmp_ctr), ch2_ready, fmt='b-', lw=1.) # ax[ifil].text(0.1, 0.9, 'Points per scan: '+str(tmp_pps), transform=ax[ifil].transAxes) # ax[ifil].text(0.1, 0.85, 'Scans: '+str(tmp_nscan), transform=ax[ifil].transAxes) if ifil==0: tmp_ctr = 0.5 * (min(frq_ready) + max(frq_ready)) ax[ifil].text(0.1, 0.95, f'{tmp_ctr:.3f}'+ ' MHz', transform=ax[ifil].transAxes) # if min(synth_uniq)==0.: # ax[ifil].text(0.1, 0.8, 'UNFINISHED SCANS', transform=ax[ifil].transAxes) ax[ifil].set_xlabel('Frequency detuning, MHz') ready_data.set(tmp_processed) return fig @render.download(filename='spectra.zip') def save_spectra(): with io.BytesIO() as buf: # this creates in-memory buffer where zip archive is stored # without storing file on the disk testzip = zipfile.ZipFile(buf, 'w') # create a ZipFile object, instead of filename use buffer object name tmp_names = names.get() # filenames specified in the list of experimental conditions tmp_data = ready_data.get() # processed recordings for i in range(len(tmp_names)): current_name = input.prefix_out()+tmp_names[i][:-3]+input.extens_out() # add prefix and change extension of the filename current_df = pd.DataFrame(tmp_data[i]) # load recording into dataframe (i don't remember why so, but this works best anyway) testzip.writestr(current_name, current_df.to_string(header=False, index=False)) # write recording in a file within the archive testzip.close() yield buf.getvalue() # return the buffer content as file to download @render.download(filename='aux_data.txt') def save_aux(): tmp_names = names.get() tmp_names_out = [input.prefix_out()+tmp_name[:-3]+input.extens_out() for tmp_name in tmp_names] # add prefix and change extension of the filename aux_data = np.zeros((len(tmp_names), 6)) tmp_aux = aux_df.get() p_total = tmp_aux['P'].values exp_pself = np.zeros_like(p_total) exp_pfor = np.zeros_like(p_total) if input.is_foreign(): if input.p_self() == 0.0: exp_pself[:] = p_total[0] else: exp_pself[:] = input.p_self() exp_pfor[:] = p_total[:] - exp_pself[:] else: exp_pself[:] = p_total[:] aux_data[:, 0] = exp_pself aux_data[:, 1] = exp_pfor aux_data[:, 2] = tmp_aux['T'].values aux_data[:, 3] = tmp_aux['dev'].values aux_data[:, 4] = 1 aux_data[:, 5] = input.cell_length() for ifil in range(len(tmp_names_out)): yield ' '.join([tmp_names_out[ifil].ljust(20)] + [('%7.3f' % elem).ljust(7) for elem in aux_data[ifil,0:4]] + ['%1i' % aux_data[ifil, 4],'%6.3f' % aux_data[ifil, 5]] + ['\n']) app = App(app_ui, server)