Spaces:
Running
Running
Commit ·
4f58ef1
0
Parent(s):
Initial commit
Browse files- .gitignore +7 -0
- Dockerfile +13 -0
- __pycache__/app.cpython-311.pyc +0 -0
- __pycache__/utils_process.cpython-311.pyc +0 -0
- app.py +227 -0
- requirements.txt +12 -0
- run.bat +1 -0
- utils_process.py +20 -0
.gitignore
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.idea/
|
| 2 |
+
venv/
|
| 3 |
+
_aux/
|
| 4 |
+
Thumbs.db
|
| 5 |
+
__pycache__/
|
| 6 |
+
*.f90
|
| 7 |
+
*.iml
|
Dockerfile
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.9
|
| 2 |
+
|
| 3 |
+
WORKDIR /code
|
| 4 |
+
|
| 5 |
+
COPY ./requirements.txt /code/requirements.txt
|
| 6 |
+
|
| 7 |
+
RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
|
| 8 |
+
|
| 9 |
+
COPY . .
|
| 10 |
+
|
| 11 |
+
EXPOSE 7860
|
| 12 |
+
|
| 13 |
+
CMD ["shiny", "run", "app.py", "--reload", "--host", "0.0.0.0", "--port", "7860"]
|
__pycache__/app.cpython-311.pyc
ADDED
|
Binary file (17.6 kB). View file
|
|
|
__pycache__/utils_process.cpython-311.pyc
ADDED
|
Binary file (1.6 kB). View file
|
|
|
app.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from shiny.types import FileInfo, NavSetArg
|
| 2 |
+
from shiny import *
|
| 3 |
+
import numpy as np
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import json
|
| 6 |
+
import zipfile
|
| 7 |
+
import io
|
| 8 |
+
import math
|
| 9 |
+
from matplotlib import pyplot as plt
|
| 10 |
+
import pandas as pd
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
from utils_process import multipass_average, rotate_data, frq_recalc
|
| 14 |
+
|
| 15 |
+
app_ui = ui.page_fluid(
|
| 16 |
+
ui.panel_title("RAD data processing"),
|
| 17 |
+
ui.layout_sidebar(ui.panel_sidebar(ui.input_file("aux_data", "Load recordings info", accept=["*.*"], multiple=False)),
|
| 18 |
+
ui.panel_main(ui.output_ui("aux_data_show"))
|
| 19 |
+
),
|
| 20 |
+
ui.row(ui.column(4, ui.input_file("input_files", "Load your spectra", accept=["*.*"], multiple=True)),
|
| 21 |
+
ui.column(4, ui.input_text("text_comment", "Commented lines", value="//"))
|
| 22 |
+
),
|
| 23 |
+
ui.row(ui.column(3, ui.input_numeric("pv_rows", "Rows", 1, min=1, max=10, step=1)),
|
| 24 |
+
ui.column(3, ui.input_numeric("pv_cols", "Cols", 6, min=1, max=10, step=1))
|
| 25 |
+
),
|
| 26 |
+
ui.input_action_button("b_preview", "Preview raw data"),
|
| 27 |
+
ui.output_plot("preview_spectrum"),
|
| 28 |
+
ui.row(ui.column(4, ui.input_numeric("n_harm", "N harmonic", 20, min=1, max=50, step=1)),
|
| 29 |
+
ui.column(4, ui.input_numeric("ifrq", "IF value, MHz", 350.)),
|
| 30 |
+
),
|
| 31 |
+
ui.input_numeric("cell_length", 'Cell length, cm', 10.),
|
| 32 |
+
ui.input_switch("s_normalize", "Normalize data"),
|
| 33 |
+
|
| 34 |
+
ui.input_action_button("b_view_processed", "Preview processed data"),
|
| 35 |
+
ui.output_plot("preview_processed"),
|
| 36 |
+
ui.row(ui.column(4, ui.input_text("prefix_out", "Output file prefix", value="")),
|
| 37 |
+
ui.column(4, ui.input_text("extens_out", "Output file extension", value="txt"))),
|
| 38 |
+
ui.row(ui.column(4, ui.input_switch("is_foreign", "Foreign broadening")),
|
| 39 |
+
ui.column(4, ui.input_numeric("p_self", "Self pressure", 0.000))),
|
| 40 |
+
ui.row(ui.column(4, ui.download_button("save_spectra", "Save spectra")),
|
| 41 |
+
ui.column(4, ui.download_button("save_aux", "Save aux data")))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def server(input, output, session):
|
| 50 |
+
|
| 51 |
+
raw_data: reactive.Value[list] = reactive.Value([])
|
| 52 |
+
names: reactive.Value[list] = reactive.Value([])
|
| 53 |
+
f_aux = reactive.Value(False) # flag that aux info loaded
|
| 54 |
+
nrecs = reactive.Value(0)
|
| 55 |
+
ready_data: reactive.Value[list] = reactive.Value([])
|
| 56 |
+
aux_df: pd.DataFrame = reactive.Value()
|
| 57 |
+
aux_df_out: pd.DataFrame = reactive.Value()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
# transfer data from loaded files into globally visible variables
|
| 61 |
+
@reactive.Effect
|
| 62 |
+
@reactive.event(input.input_files)
|
| 63 |
+
def _():
|
| 64 |
+
tmp_recs = []
|
| 65 |
+
tmp_names = []
|
| 66 |
+
f: list[FileInfo] = input.input_files()
|
| 67 |
+
# f has fields: name, size, type, datapath
|
| 68 |
+
for ifil in range(len(f)):
|
| 69 |
+
tmp_names.append(f[ifil]["name"])
|
| 70 |
+
cur_data = np.loadtxt(f[ifil]["datapath"], comments=input.text_comment())
|
| 71 |
+
tmp_recs.append(cur_data)
|
| 72 |
+
raw_data.set(tmp_recs)
|
| 73 |
+
nrecs.set(len(f))
|
| 74 |
+
names.set(tmp_names)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
@output
|
| 78 |
+
@render.ui
|
| 79 |
+
def aux_data_show():
|
| 80 |
+
if input.aux_data() is None:
|
| 81 |
+
return 'No recording conditions data loaded'
|
| 82 |
+
aux_file: list[FileInfo] = input.aux_data()
|
| 83 |
+
aux_names = ['File', 'P', 'T', 'dev']
|
| 84 |
+
aux_out = pd.read_csv(aux_file[0]["datapath"], names=aux_names, delim_whitespace=True, index_col=0)
|
| 85 |
+
aux_df.set(aux_out)
|
| 86 |
+
return ui.HTML(aux_out.to_html(classes="table table-striped"))
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
@output
|
| 91 |
+
@render.plot(alt="Preview of the loaded spectra")
|
| 92 |
+
@reactive.event(input.b_preview)
|
| 93 |
+
def preview_spectrum():
|
| 94 |
+
if input.input_files() is None:
|
| 95 |
+
fig, ax = plt.subplots()
|
| 96 |
+
ax = plt.text(0.4, 0.4, "Please upload your spectra first")
|
| 97 |
+
return fig
|
| 98 |
+
fig, axs = plt.subplots(nrows=int(input.pv_rows()), ncols=int(input.pv_cols()),
|
| 99 |
+
sharex=False, sharey=True,
|
| 100 |
+
figsize=(input.pv_cols()*2.5, input.pv_rows()*2.5), squeeze=False)
|
| 101 |
+
ax = axs.flat
|
| 102 |
+
|
| 103 |
+
for ifil in range(nrecs.get()):
|
| 104 |
+
tmp_rec = raw_data.get()[ifil]
|
| 105 |
+
tmp_synth = tmp_rec[:, 0]
|
| 106 |
+
if ifil==0:
|
| 107 |
+
tmp_ctr = 0.5 * (min(tmp_synth) + max(tmp_synth))
|
| 108 |
+
ax[ifil].text(0.1, 0.95,
|
| 109 |
+
f'{tmp_ctr*1000:.3f}'+ ' MHz',
|
| 110 |
+
transform=ax[ifil].transAxes)
|
| 111 |
+
synth_uniq, synth_counts = np.unique(tmp_synth, return_counts=True)
|
| 112 |
+
tmp_pps = synth_uniq.shape[0]
|
| 113 |
+
tmp_nscan = synth_counts[0]
|
| 114 |
+
tmp_ch1 = tmp_rec[:, 1]
|
| 115 |
+
tmp_ch2 = tmp_rec[:, 2]
|
| 116 |
+
ax[ifil].errorbar((tmp_synth - tmp_ctr)*1000, tmp_ch1, fmt='ro', ms=0.5)
|
| 117 |
+
ax[ifil].errorbar((tmp_synth - tmp_ctr)*1000, tmp_ch2, fmt='bo', ms=0.5)
|
| 118 |
+
ax[ifil].text(0.1, 0.9, 'Pps: '+str(tmp_pps) if ifil==0 else str(tmp_pps), transform=ax[ifil].transAxes)
|
| 119 |
+
ax[ifil].text(0.1, 0.85, 'Scans: '+str(tmp_nscan), transform=ax[ifil].transAxes)
|
| 120 |
+
if min(synth_uniq)==0.:
|
| 121 |
+
ax[ifil].text(0.1, 0.8, 'UNFINISHED SCANS', transform=ax[ifil].transAxes)
|
| 122 |
+
ax[ifil].set_xlabel('Frequency detuning, MHz')
|
| 123 |
+
return fig
|
| 124 |
+
|
| 125 |
+
@output
|
| 126 |
+
@render.plot(alt="Preview of the processed spectra")
|
| 127 |
+
@reactive.event(input.b_view_processed)
|
| 128 |
+
def preview_processed():
|
| 129 |
+
if input.input_files() is None:
|
| 130 |
+
fig, ax = plt.subplots()
|
| 131 |
+
ax = plt.text(0.4, 0.4, "Please upload your spectra first")
|
| 132 |
+
return fig
|
| 133 |
+
fig, axs = plt.subplots(nrows=int(input.pv_rows()), ncols=int(input.pv_cols()),
|
| 134 |
+
sharex=False, sharey=True,
|
| 135 |
+
figsize=(input.pv_cols()*2.5, input.pv_rows()*2.5), squeeze=False)
|
| 136 |
+
ax = axs.flat
|
| 137 |
+
|
| 138 |
+
tmp_processed = []
|
| 139 |
+
|
| 140 |
+
for ifil in range(nrecs.get()):
|
| 141 |
+
tmp_rec = raw_data.get()[ifil]
|
| 142 |
+
tmp_synth = tmp_rec[:, 0]
|
| 143 |
+
synth_uniq, synth_counts = np.unique(tmp_synth, return_counts=True)
|
| 144 |
+
tmp_pps = synth_uniq.shape[0]
|
| 145 |
+
tmp_nscan = synth_counts[0]
|
| 146 |
+
tmp_ch1 = tmp_rec[:, 1]
|
| 147 |
+
tmp_ch2 = tmp_rec[:, 2]
|
| 148 |
+
|
| 149 |
+
frq_ready = frq_recalc(synth_uniq*1000., input.n_harm(), input.ifrq())
|
| 150 |
+
tmp_ctr = 0.5 * (min(frq_ready) + max(frq_ready))
|
| 151 |
+
ch1_ready = multipass_average(tmp_ch1, tmp_pps)
|
| 152 |
+
ch2_ready = multipass_average(tmp_ch2, tmp_pps)
|
| 153 |
+
|
| 154 |
+
if input.s_normalize():
|
| 155 |
+
ch2_ready[:] = ch2_ready[:]/max(ch1_ready)
|
| 156 |
+
ch1_ready[:] = ch1_ready[:]/max(ch1_ready)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
tmp_processed.append(np.stack((frq_ready, ch1_ready), axis=1))
|
| 160 |
+
|
| 161 |
+
ax[ifil].errorbar((frq_ready - tmp_ctr), ch1_ready, fmt='r-', lw=1.)
|
| 162 |
+
ax[ifil].errorbar((frq_ready - tmp_ctr), ch2_ready, fmt='b-', lw=1.)
|
| 163 |
+
# ax[ifil].text(0.1, 0.9, 'Points per scan: '+str(tmp_pps), transform=ax[ifil].transAxes)
|
| 164 |
+
# ax[ifil].text(0.1, 0.85, 'Scans: '+str(tmp_nscan), transform=ax[ifil].transAxes)
|
| 165 |
+
|
| 166 |
+
if ifil==0:
|
| 167 |
+
tmp_ctr = 0.5 * (min(frq_ready) + max(frq_ready))
|
| 168 |
+
ax[ifil].text(0.1, 0.95,
|
| 169 |
+
f'{tmp_ctr:.3f}'+ ' MHz',
|
| 170 |
+
transform=ax[ifil].transAxes)
|
| 171 |
+
|
| 172 |
+
# if min(synth_uniq)==0.:
|
| 173 |
+
# ax[ifil].text(0.1, 0.8, 'UNFINISHED SCANS', transform=ax[ifil].transAxes)
|
| 174 |
+
ax[ifil].set_xlabel('Frequency detuning, MHz')
|
| 175 |
+
|
| 176 |
+
ready_data.set(tmp_processed)
|
| 177 |
+
return fig
|
| 178 |
+
|
| 179 |
+
@render.download(filename='spectra.zip')
|
| 180 |
+
def save_spectra():
|
| 181 |
+
with io.BytesIO() as buf:
|
| 182 |
+
# this creates in-memory buffer where zip archive is stored
|
| 183 |
+
# without storing file on the disk
|
| 184 |
+
testzip = zipfile.ZipFile(buf, 'w') # create a ZipFile object, instead of filename use buffer object name
|
| 185 |
+
tmp_names = names.get() # filenames specified in the list of experimental conditions
|
| 186 |
+
tmp_data = ready_data.get() # processed recordings
|
| 187 |
+
for i in range(len(tmp_names)):
|
| 188 |
+
current_name = input.prefix_out()+tmp_names[i][:-3]+input.extens_out() # add prefix and change extension of the filename
|
| 189 |
+
current_df = pd.DataFrame(tmp_data[i]) # load recording into dataframe (i don't remember why so, but this works best anyway)
|
| 190 |
+
testzip.writestr(current_name, current_df.to_string(header=False, index=False)) # write recording in a file within the archive
|
| 191 |
+
testzip.close()
|
| 192 |
+
yield buf.getvalue() # return the buffer content as file to download
|
| 193 |
+
|
| 194 |
+
@render.download(filename='aux_data.txt')
|
| 195 |
+
def save_aux():
|
| 196 |
+
tmp_names = names.get()
|
| 197 |
+
tmp_names_out = [input.prefix_out()+tmp_name[:-3]+input.extens_out() for tmp_name in tmp_names] # add prefix and change extension of the filename
|
| 198 |
+
aux_data = np.zeros((len(tmp_names), 6))
|
| 199 |
+
|
| 200 |
+
tmp_aux = aux_df.get()
|
| 201 |
+
p_total = tmp_aux['P'].values
|
| 202 |
+
exp_pself = np.zeros_like(p_total)
|
| 203 |
+
exp_pfor = np.zeros_like(p_total)
|
| 204 |
+
if input.is_foreign():
|
| 205 |
+
if input.p_self() == 0.0:
|
| 206 |
+
exp_pself[:] = p_total[0]
|
| 207 |
+
else:
|
| 208 |
+
exp_pself[:] = input.p_self()
|
| 209 |
+
exp_pfor[:] = p_total[:] - exp_pself[:]
|
| 210 |
+
else:
|
| 211 |
+
exp_pself[:] = p_total[:]
|
| 212 |
+
|
| 213 |
+
aux_data[:, 0] = exp_pself
|
| 214 |
+
aux_data[:, 1] = exp_pfor
|
| 215 |
+
aux_data[:, 2] = tmp_aux['T'].values
|
| 216 |
+
aux_data[:, 3] = tmp_aux['dev'].values
|
| 217 |
+
aux_data[:, 4] = 1
|
| 218 |
+
aux_data[:, 5] = input.cell_length()
|
| 219 |
+
|
| 220 |
+
for ifil in range(len(tmp_names_out)):
|
| 221 |
+
yield ' '.join([tmp_names_out[ifil].ljust(20)]
|
| 222 |
+
+ [('%7.3f' % elem).ljust(7) for elem in aux_data[ifil,0:4]]
|
| 223 |
+
+ ['%1i' % aux_data[ifil, 4],'%6.3f' % aux_data[ifil, 5]]
|
| 224 |
+
+ ['\n'])
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
app = App(app_ui, server)
|
requirements.txt
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pip~=21.3.1
|
| 2 |
+
wheel==0.46.2
|
| 3 |
+
tornado~=6.3.1
|
| 4 |
+
setuptools~=80.7.1
|
| 5 |
+
ipywidgets~=8.0.6
|
| 6 |
+
ipysheet~=0.7.0
|
| 7 |
+
numpy~=1.24.2
|
| 8 |
+
pandas~=1.5.3
|
| 9 |
+
shiny~=0.7.0
|
| 10 |
+
shinywidgets~=0.2.0
|
| 11 |
+
matplotlib~=3.7.1
|
| 12 |
+
scipy~=1.10.1
|
run.bat
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
shiny run --reload
|
utils_process.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
|
| 3 |
+
def multipass_average(signal: np.ndarray, npts: int) -> np.ndarray:
|
| 4 |
+
npts_tot = len(signal)
|
| 5 |
+
# npass = npts_tot // npts
|
| 6 |
+
signal_avg = np.zeros(npts)
|
| 7 |
+
for i in range(npts):
|
| 8 |
+
signal_avg[i] = np.average(signal[i:npts_tot:npts])
|
| 9 |
+
return signal_avg
|
| 10 |
+
|
| 11 |
+
def rotate_data(ch_x, ch_y, ang):
|
| 12 |
+
ang_rad = ang * np.pi / 180.
|
| 13 |
+
cos_a = np.cos(ang_rad)
|
| 14 |
+
sin_a = np.sin(ang_rad)
|
| 15 |
+
r_x = ch_x[:] * cos_a + ch_y[:] * sin_a
|
| 16 |
+
r_y = -1. * ch_x[:] * sin_a + ch_y[:] * cos_a
|
| 17 |
+
return r_x, r_y
|
| 18 |
+
|
| 19 |
+
def frq_recalc(frq: np.ndarray, n_harmonic: int, inter_fr: float) -> np.ndarray:
|
| 20 |
+
return frq[:] * n_harmonic + inter_fr
|