from Packages import *
st.set_page_config(page_title="NIRS Utils", page_icon=":goat:", layout="wide")
from Modules import *
from Class_Mod.DATA_HANDLING import *
st.session_state["interface"] = st.session_state.get('interface')
if st.session_state["interface"] == 'simple':
hide_pages("Predictions")
################################### Data Loading and Visualization ########################################
col2, col1 = st.columns([3, 1])
col1.header("Data Loading", divider='blue')
col2.header("Spectral Data Visualization", divider='blue')
## Preallocation of data structure
spectra = pd.DataFrame
meta_data = pd.DataFrame
selected_samples = pd.DataFrame
# loader for datafile
data_file = col1.file_uploader("Load NIRS Data", type=["csv","dx"], help=" :mushroom: select a csv matrix with samples as rows and lambdas as columns", key=5)
if data_file:
# Retrieve the extension of the file
test = data_file.name[data_file.name.find('.'):]
## Load .csv file
if test== '.csv':
with col1:
# Select list for CSV delimiter
psep = st.selectbox("Select csv separator - _detected_: " + str(find_delimiter('data/'+data_file.name)), options=[";", ","], index=[";", ","].index(str(find_delimiter('data/'+data_file.name))), key=9)
# Select list for CSV header True / False
phdr = st.selectbox("indexes column in csv? - _detected_: " + str(find_col_index('data/'+data_file.name)), options=["no", "yes"], index=["no", "yes"].index(str(find_col_index('data/'+data_file.name))), key=31)
if phdr == 'yes':
col = 0
else:
col = False
imp = pd.read_csv(data_file, sep=psep, index_col=col)
# spectra = col_cat(imp)[0]
# meta_data = col_cat(imp)[1]
spectra, meta_data = col_cat(imp)
st.success("The data have been loaded successfully", icon="✅")
## Load .dx file
elif test == '.dx':
# Create a temporary file to save the uploaded file
with NamedTemporaryFile(delete=False, suffix=".dx") as tmp:
tmp.write(data_file.read())
tmp_path = tmp.name
with col1:
_, spectra, meta_data = read_dx(file = tmp_path)
st.success("The data have been loaded successfully", icon="✅")
os.unlink(tmp_path)
## Visualize spectra
if not spectra.empty:
with col2:
fig = plot_spectra(spectra)
st.pyplot(fig)
############################## Exploratory data analysis ###############################
container2 = st.container(border=True)
container2.header("Exploratory Data Analysis-Multivariable Data Analysis", divider='blue')
scores, loadings, pc = st.columns([2, 3, 0.5])
influence, hotelling, qexp = st.columns([2, 2, 1])
dim_red_methods=['', 'PCA','UMAP', 'NMF'] # List of dimensionality reduction algos
cluster_methods = ['', 'Kmeans','HDBSCAN', 'AP'] # List of clustering algos
dr_model = None # dimensionality reduction model
cl_model = None # clustering model
# Dimensionality reduction
t = pd.DataFrame # scores
p = pd.DataFrame # loadings
labels = []
if not spectra.empty:
dim_red_method = pc.selectbox("Dimensionality reduction techniques: ", options = dim_red_methods, key = 37)
clus_method = pc.selectbox("Clustering techniques: ", options = cluster_methods, key = 38)
xc = standardize(spectra)
if dim_red_method == dim_red_methods[1]:
dr_model = LinearPCA(xc, Ncomp=5)
elif dim_red_method == dim_red_methods[2]:
if not meta_data.empty:
filter = meta_data.columns[1:]
col = pc.selectbox('Supervised UMAP by:', options= filter, key=108)
supervised = meta_data[col]
else:
supervised = None
dr_model = Umap(data_import = imp, numerical_data = MinMaxScale(spectra), cat_data = supervised)
if dr_model:
axis1 = pc.selectbox("x-axis", options = dr_model.scores_.columns, index=0)
axis2 = pc.selectbox("y-axis", options = dr_model.scores_.columns, index=1)
axis3 = pc.selectbox("z-axis", options = dr_model.scores_.columns, index=2)
t = pd.concat([dr_model.scores_.loc[:,axis1], dr_model.scores_.loc[:,axis2], dr_model.scores_.loc[:,axis3]], axis = 1)
# clustering
if not t.empty:
tcr = standardize(t)
# Clustering
if clus_method == cluster_methods[1]:
ncluster = scores.number_input(min_value=2, max_value=30, value=3, label = 'Select the desired number of clusters')
cl_model = Sk_Kmeans(tcr, max_clusters = 30)
fig2 = px.scatter(cl_model.inertia_.T, y = 'inertia')
scores.plotly_chart(fig2)
data, labels = cl_model.fit_optimal(nclusters = ncluster)
elif clus_method == cluster_methods[2]:
optimized_hdbscan = Hdbscan(dr_model.scores_raw_)
labels, hdbscan_score = optimized_hdbscan.HDBSCAN_scores_
##### Plots
## Scores
if not t.empty:
with scores:
st.write('Scores plot')
# scores plot with clustering
if list(labels) and meta_data.empty:
fig = px.scatter_3d(tcr, x=axis1, y=axis2, z = axis3, color = labels)
# scores plot with metadata
elif len(list(labels)) == 0 and not meta_data.empty:
filter = meta_data.columns[1:]
col = st.selectbox('Group by:', options= filter)
if col == 0:
fig = px.scatter_3d(tcr, x=axis1, y=axis2, z = axis3)
else:
fig = px.scatter_3d(tcr, x=axis1, y=axis2, z = axis3, color = list(map(str.lower,meta_data[col])) )
# color with scores and metadata
elif len(list(labels)) > 0 and not meta_data.empty:
if clus_method in cluster_methods[1:]:
filter = ['None', clus_method]
filter.extend(meta_data.columns[1:])
else:
filter = meta_data.columns[1:].insert(0,'None')
col = st.selectbox('Group by:', options= filter)
if col == "None":
fig = px.scatter_3d(tcr, x=axis1, y=axis2, z = axis3)
elif col == clus_method:
fig = px.scatter_3d(tcr, x=axis1, y=axis2, z = axis3, color = labels)
else:
fig = px.scatter_3d(tcr, x=axis1, y=axis2, z = axis3, color = list(map(str.lower,meta_data[col])))
else:
fig = px.scatter_3d(tcr, x=axis1, y=axis2, z = axis3)
fig.update_traces(marker=dict(size=4))
st.plotly_chart(fig)
if not spectra.empty:
if dim_red_method == dim_red_methods[1]:
with loadings:
st.write('Loadings plot')
p = dr_model.loadings_
pp = pd.concat([p, pd.DataFrame(np.arange(p.shape[0]), index=p.index, columns=['wl'])], axis =1)
df1 = pp.melt(id_vars="wl")
fig = px.line(df1, x = 'wl', y = 'value', color='variable')
fig.update_layout(legend=dict(x=1, y=0,font=dict(family="Courier", size=12, color="black"),
bordercolor="Black", borderwidth=2))
st.plotly_chart(fig, use_container_width = True)
with influence:
st.write('Influence plot')
ax1 = st.selectbox("Component", options=dr_model.scores_.columns, index=3)
leverage = dr_model.leverage_
residuals = dr_model.residuals_
fig = px.scatter(x=leverage[ax1], y=residuals[ax1], color = leverage[ax1]*residuals[ax1]).update_layout(xaxis_title="Leverage",yaxis_title="Residuals")
st.plotly_chart(fig)
with hotelling:
st.write('T²-Hotelling vs Q residuals plot')
hotelling = dr_model.hotelling_
ax2 = st.selectbox("Component", options=dr_model.scores_.columns, index=4)
hotelling = dr_model.hotelling_
fig = px.scatter(t, x=hotelling[ax2], y=residuals[ax2]).update_layout(xaxis_title="T²",yaxis_title="Residuals")
st.plotly_chart(fig)