From 537081b5f25b087a2269239b98036aa0d35c06f2 Mon Sep 17 00:00:00 2001
From: barthes <nicolas.barthes@cefe.cnrs.fr>
Date: Thu, 4 Apr 2024 14:45:56 +0200
Subject: [PATCH] updated requirements.txt, cleanup of tbd functions &
 Miscellaneous/ Download button function

---
 Class_Mod/LWPLSR_.py                          |  2 +-
 Class_Mod/Miscellaneous.py                    | 10 +++-
 Modules.py                                    |  5 +-
 Packages.py                                   |  9 ++--
 app.py                                        | 35 ++++++--------
 ...h_on_Xcal.csv_and_Ycal.csv_data_pickle.csv | 46 +++++++++----------
 requirements.txt                              |  5 +-
 7 files changed, 55 insertions(+), 57 deletions(-)

diff --git a/Class_Mod/LWPLSR_.py b/Class_Mod/LWPLSR_.py
index 3f13339..e3406d6 100644
--- a/Class_Mod/LWPLSR_.py
+++ b/Class_Mod/LWPLSR_.py
@@ -15,4 +15,4 @@ def model_LWPLSR(xcal_csv, ycal_csv, sep, hdr):
     X_train, y_train, X_test, y_test = x[train_index], y[train_index], x[test_index], y[test_index]
     st.write("Size of train and test sets: train " + str(X_train.shape) + ' ' + str(y_train.shape) + ' / test ' + str(X_test.shape) + ' ' + str(y_test.shape))
 
-    Jchemo.lwplsr(X_train, y_train, nlvdis=4, metric = eucl, k = 10)
\ No newline at end of file
+    # Jchemo.lwplsr(X_train, y_train, nlvdis=4, metric = eucl, k = 10)
\ No newline at end of file
diff --git a/Class_Mod/Miscellaneous.py b/Class_Mod/Miscellaneous.py
index bb7b101..1ea7dde 100644
--- a/Class_Mod/Miscellaneous.py
+++ b/Class_Mod/Miscellaneous.py
@@ -2,6 +2,7 @@ from Packages import *
     
 # local CSS
 ## load the custom CSS in the style folder
+@st.cache_data
 def local_css(file_name):
     with open(file_name) as f:
         st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
@@ -20,6 +21,7 @@ def prediction(NIRS_csv, qsep, qhdr, model):
     return Y_preds
 
 
+@st.cache_data
 def reg_plot( meas, pred):
     fig, ax = plt.subplots(figsize = (12,4))
     sns.regplot(x = meas[0] , y = pred[0], color='blue', label = 'Calib')
@@ -31,6 +33,7 @@ def reg_plot( meas, pred):
     plt.legend()
     plt.margins(0)
 
+@st.cache_data
 def resid_plot( meas, pred):
     fig, ax = plt.subplots(figsize = (12,4))
     sns.residplot(x = meas[0], y = pred[0], color='blue', label = 'Calib')
@@ -38,4 +41,9 @@ def resid_plot( meas, pred):
     sns.residplot(x = meas[2], y = pred[2], color='green', label = 'Test')
     ax.set_ylabel('Residuals')
     ax.set_xlabel('Predicted values')
-    plt.legend()
\ No newline at end of file
+    plt.legend()
+
+# function that create a download button - needs the data to save and the file name to store to
+def download_results(data, export_name):
+    with open(data) as f:
+        st.download_button('Download Results', f, export_name)
\ No newline at end of file
diff --git a/Modules.py b/Modules.py
index 90af64d..9f80e67 100644
--- a/Modules.py
+++ b/Modules.py
@@ -1,7 +1,4 @@
 from Class_Mod import LinearPCA, umap_maker, find_col_index, PinardPlsr, model_LWPLSR, list_files, metrics, TpeIpls, reg_plot, resid_plot, Sk_Kmeans
 # find_col_index
 
-from Class_Mod.Miscellaneous import  prediction
-from urllib.request import urlretrieve
-
-#
\ No newline at end of file
+from Class_Mod.Miscellaneous import prediction, download_results
diff --git a/Packages.py b/Packages.py
index 3fdabba..b2de7fe 100644
--- a/Packages.py
+++ b/Packages.py
@@ -19,8 +19,8 @@ from sklearn.cluster import KMeans
 #import hdbscan
 
 # Modelling
-import julia
-from julia import Jchemo
+# import julia
+# from julia import Jchemo
 
 from pinard import utils
 from pinard import preprocessing as pp
@@ -53,8 +53,7 @@ import json
 
 # save models
 import joblib
-import pickle as pkl
+# import pickle as pkl
 
-
-from hyperopt import fmin, hp, tpe, Trials, space_eval, STATUS_OK, anneal
+# from hyperopt import fmin, hp, tpe, Trials, space_eval, STATUS_OK, anneal
 st.set_option('deprecation.showPyplotGlobalUse', False)
diff --git a/app.py b/app.py
index a7dc627..122038f 100644
--- a/app.py
+++ b/app.py
@@ -3,13 +3,9 @@
 from Packages import *
 st.set_page_config(page_title="NIRS Utils", page_icon=":goat:", layout="wide")
 from Modules import *
-from Packages import * 
-from Class_Mod.DATA_HANDLING import * 
-
-# graphical delimiter
-st.write("---")
-
+from Class_Mod.DATA_HANDLING import *
 
+# st.write('---')
 # load images for web interface
 img_sselect = Image.open("images\sselect.JPG")
 img_general = Image.open("images\general.JPG")
@@ -26,7 +22,7 @@ with st.sidebar:
 with st.container():
     st.subheader("Plateforme d'Analyses Chimiques pour l'Ecologie-PACE :goat:")
     st.title("NIRS Utils")
-    st.write("Samples selection, Predictive Modelling, and Predictions making using [Pinard](https://github.com/GBeurier/pinard) and PACE NIRS Database.")
+    st.write("Samples selection (PCA, [UMAP](https://umap-learn.readthedocs.io/en/latest/how_umap_works.html), ...), Predictive Modelling ([Pinard](https://github.com/GBeurier/pinard), [LWPLSR](https://doi.org/10.1002/cem.3209), ...), and Predictions using your data (CSV or DX files) and/or PACE NIRS Database.")
     #st.image(img_general)
 
 
@@ -238,7 +234,7 @@ with container2:
 
 
 
-        # Export the model with pickle or joblib
+        # Export the model with joblib
         if regression_algo != '':
             M1.write("-- Performance metrics --")
             M1.dataframe(Reg.metrics_)
@@ -275,14 +271,14 @@ with st.container():
     st.write("Predict chemical values from NIRS")
     model_column, space, file_column= st.columns((2, 1, 1))
     NIRS_csv = file_column.file_uploader("Select NIRS Data to predict", type="csv", help=" :mushroom: select a csv matrix with samples as rows and lambdas as columns")
-    export_name = './data/predictions/Predictions_of_'
+    export_folder = './data/predictions/'
+    export_name = 'Predictions_of_'
     if NIRS_csv:
         export_name += str(NIRS_csv.name[:-4])
         qsep = file_column.selectbox("Select csv separator - _detected_: " + str(find_delimiter('data/'+NIRS_csv.name)), options=[";", ","], index=[";", ","].index(str(find_delimiter('data/'+NIRS_csv.name))), key=2)
         qhdr = file_column.selectbox("indexes column in csv? - _detected_: " + str(find_col_index('data/'+NIRS_csv.name)), options=["no", "yes"], index=["no", "yes"].index(str(find_col_index('data/'+NIRS_csv.name))), key=3)
-        
-        
-        # Load the model with pickle or joblib
+
+        # Load the model with joblib
         model_column.write("Load your saved predictive model")
         model_name_import = model_column.selectbox('Choose file:', options=os.listdir('data/models/'), key = 21)
         if model_name_import != ' ':
@@ -297,14 +293,9 @@ with st.container():
         # use prediction function from application_functions.py to predict chemical values
         result = prediction(NIRS_csv, qsep, qhdr, model_loaded)
         st.write('Predicted values are: ')
-        st.dataframe(result.T)
-        pd.DataFrame(result).to_csv(export_name + '.csv')
-        st.write('Predictions exported to ' + export_name + '.csv')
-        # export to local drive
-        url = ('http://localhost:8501' + export_name[1:] + '.csv')
-        filename = export_name + '.csv'
-        urlretrieve(url, filename)
+        st.dataframe(result)
+        pd.DataFrame(result).to_csv(export_folder + export_name + '.csv')
+        # export to local drive - Download
+        download_results(export_folder + export_name + '.csv', export_name + '.csv')
         # create a report with information on the prediction
-        ## see https://stackoverflow.com/a/59578663
-    if type(result) is list:
-        st.write(result)
\ No newline at end of file
+        ## see https://stackoverflow.com/a/59578663
\ No newline at end of file
diff --git a/data/predictions/Predictions_of_Xcal_with_model_jh_on_Xcal.csv_and_Ycal.csv_data_pickle.csv b/data/predictions/Predictions_of_Xcal_with_model_jh_on_Xcal.csv_and_Ycal.csv_data_pickle.csv
index 01c9099..2434424 100644
--- a/data/predictions/Predictions_of_Xcal_with_model_jh_on_Xcal.csv_and_Ycal.csv_data_pickle.csv
+++ b/data/predictions/Predictions_of_Xcal_with_model_jh_on_Xcal.csv_and_Ycal.csv_data_pickle.csv
@@ -8,7 +8,7 @@
 6,45.33593131421402
 7,46.84716401517402
 8,47.55778099910814
-9,43.66400920123191
+9,43.664009201231906
 10,46.389948147643885
 11,46.93134883054068
 12,44.68002922429634
@@ -17,7 +17,7 @@
 15,45.11320879514305
 16,45.8935707706589
 17,46.07742471331517
-18,46.11265896577673
+18,46.11265896577674
 19,46.27604577435942
 20,48.02201720885121
 21,45.99579895733129
@@ -37,7 +37,7 @@
 35,46.54179329103721
 36,44.80908398747734
 37,47.308088930673506
-38,46.9631078693342
+38,46.96310786933421
 39,46.04629576575285
 40,45.470485120803296
 41,47.80573031577645
@@ -71,7 +71,7 @@
 69,45.575702172801265
 70,47.6980790423098
 71,45.533416530723066
-72,45.47488420381561
+72,45.47488420381562
 73,44.55515653767657
 74,44.39530657933428
 75,46.676915986327096
@@ -88,14 +88,14 @@
 86,47.091404503230834
 87,46.619394266773085
 88,45.06345107435841
-89,44.715128053554594
-90,43.92052613415367
+89,44.71512805355459
+90,43.920526134153675
 91,46.07078260158221
 92,43.333601792345256
 93,46.37279879543664
 94,45.29533969506365
 95,47.65581989149592
-96,47.26124233132717
+96,47.261242331327175
 97,45.43754723787828
 98,47.558428644932825
 99,45.43891036793652
@@ -127,14 +127,14 @@
 125,46.117492034670626
 126,44.64106025415083
 127,45.48948771021548
-128,46.81034582829701
+128,46.81034582829702
 129,45.64425368921125
 130,45.5529990463294
 131,47.70474912071469
 132,45.68681659656469
 133,46.27003325246771
 134,45.345787878984545
-135,46.70370265288267
+135,46.703702652882676
 136,46.045030623620875
 137,45.27480869528746
 138,45.53995144880514
@@ -148,12 +148,12 @@
 146,46.547667743908406
 147,44.18893017542523
 148,46.05166221986108
-149,46.91993536042849
+149,46.9199353604285
 150,45.223809725359075
 151,45.643290374726675
 152,46.96338566255745
 153,46.30388683306703
-154,47.73870866596065
+154,47.738708665960665
 155,46.387277357483974
 156,45.1538455213669
 157,46.372628630709514
@@ -200,7 +200,7 @@
 198,45.88322456266033
 199,45.72205581327608
 200,45.31606441873041
-201,46.83763175995848
+201,46.837631759958484
 202,46.16584089134833
 203,47.602817232982694
 204,44.52519206870877
@@ -234,8 +234,8 @@
 232,45.837193699345654
 233,46.869311842994456
 234,46.58738309513394
-235,46.0985553856456
-236,43.171676856058454
+235,46.098555385645604
+236,43.17167685605845
 237,45.26438253565962
 238,46.02191992621766
 239,46.50637514957429
@@ -263,12 +263,12 @@
 261,44.8978471964728
 262,47.22097732127542
 263,46.527228181810486
-264,46.589932517692766
+264,46.58993251769277
 265,47.23597864342218
 266,46.0295360959636
-267,46.05531848450945
+267,46.055318484509456
 268,47.22572842162108
-269,45.78999301614902
+269,45.789993016149026
 270,45.752067568799376
 271,46.15529597529264
 272,45.640102199613125
@@ -280,7 +280,7 @@
 278,46.128968478562015
 279,45.32773315134333
 280,46.55924089553013
-281,46.745561634451676
+281,46.74556163445168
 282,45.425561122539975
 283,45.1689096828214
 284,45.74714210773718
@@ -307,7 +307,7 @@
 305,45.5084676952205
 306,46.20812641249573
 307,46.58463198165516
-308,47.36575447448282
+308,47.365754474482834
 309,45.8732669263878
 310,45.684525233204994
 311,46.5417091597917
@@ -324,9 +324,9 @@
 322,46.81132687453868
 323,46.235693184472815
 324,45.73787284994606
-325,46.7078094065248
+325,46.70780940652481
 326,46.738999886685605
-327,46.188008062786054
+327,46.18800806278606
 328,45.990317923515725
 329,45.52791818878071
 330,45.09572722884058
@@ -347,8 +347,8 @@
 345,46.15053642203064
 346,46.45225840594316
 347,45.52401656865006
-348,46.220843786773564
-349,47.23874532242078
+348,46.22084378677357
+349,47.23874532242079
 350,46.4990213308299
 351,45.887652079472275
 352,45.790247414108165
diff --git a/requirements.txt b/requirements.txt
index df3cab0..9a71166 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,7 @@ watchdog>=2.1.8
 pinard>=1.0
 julia>=0.6.2
 plotly>=5.20.0
-pyodbc>=5.1.0
\ No newline at end of file
+pyodbc>=5.1.0
+matplotlib>=3.8.4
+seaborn>=0.13.2
+hyperopt>=0.2.7
\ No newline at end of file
-- 
GitLab