Visualisation des spectres prétraités à l'aide d'une échelle de couleurs...

Visualisation des spectres prétraités à l'aide d'une échelle de couleurs continue basée sur les valeurs y
Preprocessing parameters vizualization

src/Class_Mod/RegModels.py

@@ -12,8 +12,8 @@ class Regmodel(object):
         self._yc, self._ycv, self._yt = None, None, None
         self._cv_df = pd.DataFrame
         self._nfolds = nfolds
-        self.bands = pd.DataFrame()
-        self.important_features = pd.DataFrame()
+        self._selected_bands = pd.DataFrame(index = ['from', 'to'])
+        self.important_features = None
         self._hyper_params = {'polyorder': hp.choice('polyorder', [0, 1, 2]),
                             'deriv': hp.choice('deriv', [0, 1, 2]),
                             'window_length': hp.choice('window_length', [15, 21, 27, 33]),
@@ -38,6 +38,10 @@ class Regmodel(object):
     def test_data_(self):
         return [self._xt, self._ytest]
 
+    @property
+    def pretreated_spectra_(self):
+        return self.pretreated
+
     @property
     def get_params_(self):
        return self._hyper_params
@@ -46,8 +50,19 @@ class Regmodel(object):
        pass
     
     @property
-    def best_hyperparams(self): 
-       return self._best
+    def best_hyperparams_(self):
+        return self._best
+    @property
+    def best_hyperparams_print(self):
+        if self._best['scatter'] == 'Snv':
+            a = 'Standard Normal Variate (SNV)'
+
+        elif self._best['scatter'] == 'No_transformation':
+            a = " No transformation was performed"
+
+        SG = f'- Savitzky-Golay derivative parameters (Window_length:{self._best['window_length']};  polynomial order: {self._best['polyorder']};  Derivative order : {self._best['deriv']})'
+        Norm = f'- Spectral Normalization: {a}'
+        return SG+"\n"+Norm
     
     @property
     def model_(self):
@@ -67,6 +82,9 @@ class Regmodel(object):
     @property
     def important_features_(self):
         return self.important_features
+    @property
+    def selected_features_(self):
+        return self._selected_bands
   
 ###########################################    #########################################
 class Plsr(Regmodel):
@@ -108,7 +126,7 @@ class Plsr(Regmodel):
             self._yt = Model.predict(x2[1])
             self._model = Model
             self._best = params
-            self.x2 = x2[0]
+            self.pretreated = pd.DataFrame(x2[0])
         return score
 
 
@@ -117,13 +135,13 @@ class TpeIpls(Regmodel):
     def __init__(self, train: [pd.DataFrame, pd.DataFrame], test: [pd.DataFrame, pd.DataFrame], n_iter = 10, n_intervall = 5):
         self.n_intervall = n_intervall
         self.n_arrets = self.n_intervall*2
-        self.bands = pd.DataFrame()
-        self.bands.index = ['from', 'to']
+        
         
         r = {'n_components': hp.randint('n_components', 2,20)}
         r.update({f'v{i}': hp.randint(f'v{i}', 0, train[0].shape[1]) for i in range(1,self.n_arrets+1)})
 
         super().__init__(train, test, n_iter, add_hyperparams = r)
+        
         ### parameters in common
         
     def objective(self, params):
@@ -148,10 +166,16 @@ class TpeIpls(Regmodel):
         params['deriv'], params['polyorder'], params['window_length']  = a, b, c
         x2 = [savgol_filter(x1[i], polyorder=params['polyorder'], deriv=params['deriv'], window_length = params['window_length']) for i in range(2)]
         # print(x2)
-
+        
         # ## Modelling
-        Model = PLSRegression(scale = False, n_components = params['n_components'])
-        self._cv_df = KF_CV().process(model = Model, x = x2[0][:,id], y = self._ytrain, n_folds = self._nfolds)
+        try:
+            Model = PLSRegression(scale = False, n_components = params['n_components'])
+            self._cv_df = KF_CV().process(model = Model, x = x2[0][:,id], y = self._ytrain, n_folds = self._nfolds)
+        except ValueError as ve:
+            params["n_components"] = 1
+            Model = PLSRegression(scale = False, n_components = params['n_components'])
+            self._cv_df = KF_CV().process(model = Model, x = x2[0][:,id], y = self._ytrain, n_folds = self._nfolds)
+
         self._cv_df['Average'] = self._cv_df.mean(axis = 1)
         self._cv_df['S'] = self._cv_df.std(axis = 1)
         self._cv_df['CV(%)'] = self._cv_df['S'] * 100 / self._cv_df['Average']
@@ -168,12 +192,12 @@ class TpeIpls(Regmodel):
             self._yt = Model.predict(x2[1][:,id])
             self._model = Model
             self._best = params
-            self.x2 = x2[0][:,id]
+            self.pretreated = pd.DataFrame(x2[0])
             self.segments = arrays
             
             for i in range(len(self.segments)):
-                self.bands[f'band{i+1}'] = [self.segments[i][0], self.segments[i][self.segments[i].shape[0]-1]]
-            self.bands.index = ['from','to']
+                self._selected_bands[f'band{i+1}'] = [self.segments[i][0], self.segments[i][self.segments[i].shape[0]-1]]
+            self._selected_bands.index = ['from','to']
                 
         return score
     

src/pages/1-samples_selection.py

@@ -12,6 +12,9 @@ from Modules import *
 # st.markdown(bandeau_html, unsafe_allow_html=True)
 add_header()
 
+# path = os.path.dirname(os.path.abspath(__file__)).replace('\\','/')
+# css_file = path[:path.find('/pages')]+'/style'
+# local_css(css_file +"/style_model.css")
 local_css(css_file / "style_model.css")
 
 st.session_state["interface"] = st.session_state.get('interface')

src/pages/2-model_creation.py

@@ -5,8 +5,14 @@ from Modules import *
 from Class_Mod.DATA_HANDLING import *
 from pandas.api.types import is_float_dtype
 from Class_Mod.Miscellaneous import desc_stats
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from matplotlib.cm import ScalarMappable
 add_header()
 
+import matplotlib.pyplot as plt, mpld3
+import streamlit.components.v1 as components
+
 st.session_state["interface"] = st.session_state.get('interface')
 if st.session_state["interface"] == 'simple':
     hide_pages("Predictions")
@@ -22,8 +28,8 @@ st.markdown("Create a predictive model, then use it for predicting your target v
 st.header("I - Data visualization", divider='blue')
 M0, M00 = st.columns([1, .4])
 st.header("II - Model creation", divider='blue')
-M1, M2, M3 = st.columns([2,2,2])
-st.header("Cross_Validation")
+M1, M2 = st.columns([2 ,4])
+st.header("Cross-Validation")
 cv1, cv2 = st.columns([2,2])
 cv3 = st.container()
 
@@ -180,7 +186,7 @@ if not spectra.empty and not y.empty:
 
     elif regression_algo == reg_algo[3]:
         s = M1.number_input(label='Enter the maximum number of intervals', min_value=1, max_value=6, value=3)
-        it = M1.number_input(label='Enter the number of iterations', min_value=50, max_value=1000, value=100)
+        it = M1.number_input(label='Enter the number of iterations', min_value=2, max_value=10, value=3)
         progress_text = "The model is being created. Please wait."
             
         Reg = TpeIpls(train = [X_train, y_train], test=[X_test, y_test], n_intervall = s, n_iter=it)
@@ -189,35 +195,100 @@ if not spectra.empty and not y.empty:
         M1.progress(100, text = "The model has successfully been  created!")            
         time.sleep(1)
         reg_model = Reg.model_
-        M3.write('-- Spectral regions used for model creation --')
-        intervalls = Reg.bands.T
-        M3.table(intervalls)
-        fig, ax = plt.subplots(figsize = (12, 6))
-        X_train.mean().plot(ax = ax)
-        for i in range(s):
-            colnames = np.array(y)
-            num = {'u','i','f','c'}
-            if np.array(X_train.columns).dtype.kind in num:
-                plt.plot(X_train.columns, X_train.mean(), color = 'black')
-                ax.axvspan(X_train.columns[intervalls['from'][i]], X_train.columns[intervalls['to'][i]],
-                            color='#2a52be', alpha=0.5, lw=0)
-                plt.tight_layout()
-                plt.margins(x = 0)
-            else:
-                plt.plot(np.arange(X_train.shape[1]), X_train.mean(), color = 'black')
-                ax.axvspan(intervalls['from'][i], intervalls['to'][i], color='#2a52be', alpha=0.5, lw=0)
-                plt.tight_layout()
-                plt.margins(x = 0)
-        
-        M3.write('-- Visualization of the spectral regions used for model creation --   ')
-        M3.pyplot(fig)
+        # M3.write('-- Spectral regions used for model creation --')
+        # intervalls = Reg.bands.T
+        # M3.table(intervalls)
+        # fig, ax = plt.subplots(figsize = (12, 6))
+        # X_train.mean().plot(ax = ax)
+        # for i in range(s):
+        #     colnames = np.array(y)
+        #     num = {'u','i','f','c'}
+        #     if np.array(X_train.columns).dtype.kind in num:
+        #         plt.plot(X_train.columns, X_train.mean(), color = 'black')
+        #         ax.axvspan(X_train.columns[intervalls['from'][i]], X_train.columns[intervalls['to'][i]],
+        #                     color='#2a52be', alpha=0.5, lw=0)
+        #         plt.tight_layout()
+        #         plt.margins(x = 0)
+        #     else:
+        #         plt.plot(np.arange(X_train.shape[1]), X_train.mean(), color = 'black')
+        #         ax.axvspan(intervalls['from'][i], intervalls['to'][i], color='#2a52be', alpha=0.5, lw=0)
+        #         plt.tight_layout()
+        #         plt.margins(x = 0)
         
+        # M3.write('-- Visualization of the spectral regions used for model creation --   ')
+        # M3.pyplot(fig)
+        M2.write('-- Spectral regions used for model creation --')
+        intervalls = Reg.selected_features_.T
+        M2.table(intervalls)
+
     # elif regression_algo == reg_algo[4]:
     #     Reg = PlsR(x_train = X_train, x_test = X_test, y_train = y_train, y_test = y_test)
     #     reg_model = Reg.model_
 
         ################# Model analysis ############
     if regression_algo in reg_algo[1:]:
+        M2.write('-- Pretreated data (train) visualization and important spectral regions in the model --   ')
+
+        fig, (ax1, ax2) = plt.subplots(2,1, figsize = (12, 6))
+        fig = make_subplots(rows=3, cols=1, shared_xaxes=True, vertical_spacing=0.02)
+        # fig.append_trace(go.Scatter(x=[3, 4, 5],
+        #                             y=[1000, 1100, 1200],), row=1, col=1)
+
+        # fig.append_trace(go.Scatter(x=[2, 3, 4],
+        #                             y=[100, 110, 120],), row=2, col=1)
+
+        # fig.append_trace(go.Scatter(x=[0, 1, 2],
+        #                             y=[10, 11, 12]), row=3, col=1)
+
+        # fig.update_layout(height=600, width=600, title_text="Stacked Subplots")   
+        # a = Reg.pretreated_spectra_
+        # r = pd.concat([y_train, a], axis = 1)
+        # rr = r.melt("x")
+        # rr.columns = ['y values', 'x_axis', 'y_axis']
+        # fig = px.scatter(rr, x = 'x_axis', y = 'y_axis', color_continuous_scale=px.colors.sequential.Viridis, color = 'y values')
+        # M3.plotly_chart(fig)
+        
+        
+        from matplotlib.colors import Normalize
+        color_variable = y_train
+        norm = Normalize(vmin=color_variable.min(), vmax= color_variable.max())
+        cmap = plt.get_cmap('viridis')
+        colors = cmap(norm(color_variable.values))
+        fig, ax = plt.subplots(figsize = (10,3))
+
+        for i in range(Reg.pretreated_spectra_.shape[0]):
+            ax.plot(Reg.pretreated_spectra_.columns, Reg.pretreated_spectra_.iloc[i,:], color = colors[i])
+        sm = ScalarMappable(norm = norm, cmap = cmap)
+        cbar = plt.colorbar(sm, ax = ax)
+        # cbar.set_label('Target range') 
+        plt.tight_layout()      
+        htmlfig = mpld3.fig_to_html(fig)
+        with M2:
+            st.components.v1.html(htmlfig, height=600)
+        
+        
+
+        # X_train.mean().plot(ax = ax2)
+        # for i in range(s):
+        #     colnames = np.array(y)
+        #     num = {'u','i','f','c'}
+        #     if np.array(X_train.columns).dtype.kind in num:
+        #         plt.plot(X_train.columns, X_train.mean(), color = 'black')
+        #         ax2.axvspan(X_train.columns[intervalls['from'][i]], X_train.columns[intervalls['to'][i]],
+        #                     color='#2a52be', alpha=0.5, lw=0)
+        #         plt.tight_layout()
+        #         plt.margins(x = 0)
+        #     else:
+        #         plt.plot(np.arange(X_train.shape[1]), X_train.mean(), color = 'black')
+        #         ax2.axvspan(intervalls['from'][i], intervalls['to'][i], color='#2a52be', alpha=0.5, lw=0)
+        #         plt.tight_layout()
+        #         plt.margins(x = 0)
+        
+        # pd.DataFrame(Reg.pretreated_spectra_).plot(ax = ax1)
+        # M3.pyplot(fig)
+        
+        
+        ############
         cv2.write('-- Cross-Validation Summary--')
         cv2.write(Reg.CV_results_)
         cv2.write('-- Out-of-Fold Predictions Visualization (All in one) --')
@@ -226,9 +297,7 @@ if not spectra.empty and not y.empty:
                  color_discrete_sequence=px.colors.qualitative.G10)
         fig1.add_shape(type='line', x0 = .95 * min(Reg.cv_data_[2].loc[:,'Measured']), x1 = 1.05 * max(Reg.cv_data_[2].loc[:,'Measured']), y0 = .95 * min(Reg.cv_data_[2].loc[:,'Measured']), y1 = 1.05 * max(Reg.cv_data_[2].loc[:,'Measured']), line = dict(color='black', dash = "dash"))
         fig1.update_traces(marker_size=7, showlegend=False)
-        
         cv2.plotly_chart(fig1)
-
         fig0 = px.scatter(Reg.cv_data_[2], x ='Measured', y = 'Predicted' , trendline='ols', color='Folds', symbol="Folds", facet_col = 'Folds',facet_col_wrap=1,
                  color_discrete_sequence=px.colors.qualitative.G10, text='index', width=800, height=1000)
         fig0.update_traces(marker_size=8, showlegend=False)
@@ -240,13 +309,13 @@ if not spectra.empty and not y.empty:
         yt = Reg.pred_data_[1]
             
         #if
-        M2.write('-- Spectral preprocessing info --')
-        M2.write(Reg.best_hyperparams)
+        M1.write('-- Spectral preprocessing info --')
+        M1.write(Reg.best_hyperparams_print)
         with open("data/params/Preprocessing.json", "w") as outfile: 
-            json.dump(Reg.best_hyperparams, outfile)      
+            json.dump(Reg.best_hyperparams_, outfile)      
 
-        M2.write("-- Model performance --")
-        M2.dataframe(metrics(c = [y_train, yc], t = [y_test, yt], method='regression').scores_)
+        M1.write("-- Model performance --")
+        M1.dataframe(metrics(c = [y_train, yc], t = [y_test, yt], method='regression').scores_)
         #from st_circular_progress import CircularProgress
         #my_circular_progress = CircularProgress(label = 'Performance',value = 50, key = 'my performance',
         #                                         size = "medium", track_color = "black", color = "blue")