added UMAP to PCA dimension reduction

added 2nd k-means to ensure random samples in a cluster are not close to each other in k-means sample selection

app.py

@@ -8,7 +8,7 @@ import pandas as pd
 import plotly.express as px
 from sklearn.cluster import KMeans as km
 from sklearn.metrics import pairwise_distances_argmin_min
-from application_functions import pca_maker, model, predict, find_delimiter
+from application_functions import pca_maker, model, predict, find_delimiter, umap_maker
 
 # load images for web interface
 img_sselect = Image.open("images\sselect.JPG")
@@ -50,73 +50,90 @@ with st.container():
             col = False
 
         data_import = pd.read_csv(sselectx_csv, sep=psep, index_col=col)
+        # Select type of plot
+        plot_type=['pca','umap']
+        type_plot = settings_column.selectbox("Dimensional reduction: ", options=plot_type, key=37)
+        # compute UMAP - umap_maker in application_functions.py
+        if type_plot == 'umap':
+            pc_data, cat_cols, pc_cols = umap_maker(data_import)
         # compute PCA - pca_maker function in application_functions.py
-        pca_data, cat_cols, pca_cols = pca_maker(data_import)
+        if type_plot == 'pca':
+            pc_data, cat_cols, pc_cols = pca_maker(data_import)
         # add 2 select lists to choose which component to plot
-        pca_1 = settings_column.selectbox("First Principle Component", options=pca_cols, index=0)
-        pca_2 = settings_column.selectbox("Second Principle Component", options=pca_cols, index=1)
+        pc_1 = settings_column.selectbox("First Principle Component", options=pc_cols, index=0)
+        pc_2 = settings_column.selectbox("Second Principle Component", options=pc_cols, index=1)
         # if categorical variables exist, add 2 select lists to choose the categorical variables to color the PCA
         if cat_cols[0] == "no categories":
-            plot_pca = scatter_column.plotly_chart(px.scatter(data_frame=pca_data, x=pca_1, y=pca_2, template="simple_white", height=800, hover_name=pca_data.index, title="PCA plot of sample spectra"))
+            plot_pc = scatter_column.plotly_chart(px.scatter(data_frame=pc_data, x=pc_1, y=pc_2, template="simple_white", height=800, hover_name=pc_data.index, title="PC plot of sample spectra"))
         else:
             categorical_variable = settings_column.selectbox("Variable Select", options = cat_cols)
             categorical_variable_2 = settings_column.selectbox("Second Variable Select (hover data)", options = cat_cols)
-            plot_pca = scatter_column.plotly_chart(px.scatter(data_frame=pca_data, x=pca_1, y=pca_2, template="simple_white", height=800, color=categorical_variable, hover_data = [categorical_variable_2], hover_name=pca_data.index, title="PCA plot of sample spectra"))
-        #K-Means
-        ## K-Means choose number of clusters
-        wcss_samples = []
-        cluster_max = settings_column.slider("Max clusters (K-Means)", min_value=2, max_value=100, value=50, format="%i")
-        clusters_sample = np.arange(2, cluster_max)
-        for i in clusters_sample:
-            kmeans_samples = km(n_clusters = i, init = 'k-means++', random_state = 42)
-            kmeans_samples.fit(pca_data.loc[:,[pca_1,pca_2]])
-            wcss_samples.append(kmeans_samples.inertia_)
-        settings_column.plotly_chart(px.line(x=clusters_sample, y=wcss_samples, title="K-Means clusters nb sel", width=200))
-        ## Draw clustering
-        nb_select = settings_column.slider("Choose cluster number (K-Means)", min_value=2, max_value=cluster_max, value=5, format="%i")
-        kmeans_samples = km(n_clusters=nb_select, random_state=42)
-        kmeans_samples.fit(pca_data.loc[:,[pca_1,pca_2]])
-        # plot the pca with clustering only (no selected samples)
-        # graph = px.scatter(data_frame=pca_data, x=pca_1, y=pca_2, template="simple_white", height=800, color=kmeans_samples.labels_, hover_name=pca_data.index, title="PCA projection with K-Means Clusters")
-        # plot = scatter_column.plotly_chart(graph)
-        # choose between cluster centered sample and random samples
-        selection = settings_column.select_slider('Centered samples or random ones', options=['center','random'])
-        export = []
-        scatter_column.write("Selected samples for chemical analysis:")
-        if selection == 'center':
-            # list samples at clusters centers - Use sklearn.metrics.pairwise_distances_argmin if you want more than 1 sample per cluster
-            closest, _ = pairwise_distances_argmin_min(kmeans_samples.cluster_centers_, pca_data.loc[:,[pca_1,pca_2]])
-            scatter_column.dataframe(pca_data.loc[pca_data.index[closest],[pca_1,pca_2]], use_container_width=True)
-            export.append(pca_data.loc[pca_data.index[closest],[pca_1,pca_2]].index.T)
-            # list indexes of selected samples for colored plot
-            te = pca_data.loc[pca_data.index[closest],[pca_1,pca_2]].index.values.tolist()
-        elif selection == 'random':
-            selection_number = settings_column.number_input('How many samples per cluster?', step=1, value = 3)
-            for i in np.unique(kmeans_samples.labels_):
-                if len(pd.DataFrame(pca_data.loc[pca_data.index[kmeans_samples.labels_==i],[pca_1,pca_2]])) >= selection_number:
-                    export.append(pca_data.loc[pca_data.index[kmeans_samples.labels_==i]].sample(n=selection_number).index)
-                else:
-                    export.append(pca_data.loc[pca_data.index[kmeans_samples.labels_==i]].index)
-            # list indexes of selected samples for colored plot
-            te = []
-            for sublist in export:
-                for item in sublist:
-                    te.append(item)
-            # display a matrix of selected samples
-            scatter_column.write(pd.DataFrame(export).T)
-        # convert cluster number to text for optimized coloring
-        kmeans_samples.labels_ = kmeans_samples.labels_.astype(str)
-        for j in te:
-            kmeans_samples.labels_[pca_data.index.get_loc(j)] = 'selected'
-        # plot de pca with colored clusters and selected samples
-        graph_selected = px.scatter(data_frame=pca_data, x=pca_1, y=pca_2, template="simple_white", height=800, color=kmeans_samples.labels_, hover_name=pca_data.index, title="PCA projection with K-Means Clusters and selected samples")
-        plot = scatter_column.plotly_chart(graph_selected)
-        # button to export the names of selected samples - by cluster if random - in a csv
-        if scatter_column.button('Export'):
-            pd.DataFrame(export).T.to_csv('./data/Samples_for_Chemical_Analysis.csv')
-    else:
-        scatter_column.write("_Please Choose a file_")
-
+            plot_pc = scatter_column.plotly_chart(px.scatter(data_frame=pc_data, x=pc_1, y=pc_2, template="simple_white", height=800, color=categorical_variable, hover_data = [categorical_variable_2], hover_name=pc_data.index, title="PC plot of sample spectra"))
+        # Clustering method
+        cluster_type = ['k-means', 'umap']
+        type_cluster = settings_column.selectbox("Clustering method: ", options=cluster_type, key=38)
+        if type_cluster == 'k-means':
+            #K-Means
+            ## K-Means choose number of clusters
+            wcss_samples = []
+            cluster_max = settings_column.slider("Max clusters (K-Means)", min_value=2, max_value=100, value=50, format="%i")
+            clusters_sample = np.arange(2, cluster_max)
+            for i in clusters_sample:
+                kmeans_samples = km(n_clusters = i, init = 'k-means++', random_state = 42)
+                kmeans_samples.fit(pc_data.loc[:,[pc_1,pc_2]])
+                wcss_samples.append(kmeans_samples.inertia_)
+            settings_column.plotly_chart(px.line(x=clusters_sample, y=wcss_samples, title="K-Means clusters nb sel", width=200))
+            ## Draw clustering
+            nb_select = settings_column.slider("Choose cluster number (K-Means)", min_value=2, max_value=cluster_max, value=5, format="%i")
+            kmeans_samples = km(n_clusters=nb_select, random_state=42)
+            kmeans_samples.fit(pc_data.loc[:,[pc_1,pc_2]])
+            # plot the pc with clustering only (no selected samples)
+            # graph = px.scatter(data_frame=pc_data, x=pc_1, y=pc_2, template="simple_white", height=800, color=kmeans_samples.labels_, hover_name=pc_data.index, title="PC projection with K-Means Clusters")
+            # plot = scatter_column.plotly_chart(graph)
+            # choose between cluster centered sample and random samples
+            selection = settings_column.select_slider('Centered samples or random ones', options=['center','random'])
+            export = []
+            scatter_column.write("Selected samples for chemical analysis:")
+            if selection == 'center':
+                # list samples at clusters centers - Use sklearn.metrics.pairwise_distances_argmin if you want more than 1 sample per cluster
+                closest, _ = pairwise_distances_argmin_min(kmeans_samples.cluster_centers_, pc_data.loc[:,[pc_1,pc_2]])
+                scatter_column.dataframe(pc_data.loc[pc_data.index[closest],[pc_1,pc_2]].index, use_container_width=False)
+                export.append(pc_data.loc[pc_data.index[closest],[pc_1,pc_2]].index.T)
+                # list indexes of selected samples for colored plot
+                te = pc_data.loc[pc_data.index[closest],[pc_1,pc_2]].index.values.tolist()
+            elif selection == 'random':
+                selection_number = settings_column.number_input('How many samples per cluster?', step=1, value = 3)
+                for i in np.unique(kmeans_samples.labels_):
+                    if len(pd.DataFrame(pc_data.loc[pc_data.index[kmeans_samples.labels_==i],[pc_1,pc_2]])) >= selection_number:
+                        # export.append(pc_data.loc[pc_data.index[kmeans_samples.labels_==i]].sample(n=selection_number).index)
+                        # another k-means to cluster in 'selection_number' clusters and random to ensure the selected samples are far from each other in each cluster
+                        kmeans_selected_samples = km(n_clusters=selection_number, random_state=42)
+                        kmeans_selected_samples.fit(pc_data.loc[pc_data.index[kmeans_samples.labels_==i],[pc_1,pc_2]])
+                        closest_selected_samples, _ = pairwise_distances_argmin_min(kmeans_selected_samples.cluster_centers_, pc_data.loc[:,[pc_1,pc_2]])
+                        export.append(pc_data.loc[pc_data.index[closest_selected_samples],[pc_1,pc_2]].index)
+                    else:
+                        export.append(pc_data.loc[pc_data.index[kmeans_samples.labels_==i]].index)
+                # list indexes of selected samples for colored plot
+                te = []
+                for sublist in export:
+                    for item in sublist:
+                        te.append(item)
+                # display a matrix of selected samples
+                scatter_column.write(pd.DataFrame(export).T)
+            # convert cluster number to text for optimized coloring
+            kmeans_samples.labels_ = kmeans_samples.labels_.astype(str)
+            for j in te:
+                kmeans_samples.labels_[pc_data.index.get_loc(j)] = 'selected'
+            # plot de pc with colored clusters and selected samples
+            graph_selected = px.scatter(data_frame=pc_data, x=pc_1, y=pc_2, template="simple_white", height=800, color=kmeans_samples.labels_, hover_name=pc_data.index, title="PC projection with K-Means Clusters and selected samples")
+            plot = scatter_column.plotly_chart(graph_selected)
+            # button to export the names of selected samples - by cluster if random - in a csv
+            if scatter_column.button('Export'):
+                pd.DataFrame(export).T.to_csv('./data/Samples_for_Chemical_Analysis.csv')
+        else:
+            scatter_column.write("_Please Choose a file_")
+        if type_cluster == 'umap':
+            pass
 
 # graphical delimiter
 st.write("---")

application_functions.py

@@ -4,6 +4,7 @@ import pandas as pd
 from sklearn.decomposition import PCA
 from sklearn.preprocessing import StandardScaler
 import csv
+from umap.umap_ import UMAP
 
 # local CSS
 ## load the custom CSS in the style folder
@@ -19,8 +20,8 @@ def find_delimiter(filename):
         delimiter = sniffer.sniff(fp.read(5000)).delimiter
     return delimiter
 
-# PCA function for the Sample Selection module
-def pca_maker(data_import):
+# detection of columns categories and scaling
+def col_cat(data_import):
     # detect numerical and categorical columns in the csv
     numerical_columns_list = []
     categorical_columns_list = []
@@ -45,6 +46,25 @@ def pca_maker(data_import):
     # Scale the numerical data
     scaler = StandardScaler()
     scaled_values = scaler.fit_transform(numerical_data)
+    return numerical_data, categorical_data, scaled_values
+
+# UMAP function for the Sample Selection module
+def umap_maker(data_import):
+    numerical_data, categorical_data, scaled_values = col_cat(data_import)
+    umap_func = UMAP(random_state=42, n_neighbors=30, n_components=4)
+    umap_fit = umap_func.fit(scaled_values)
+    umap_data = umap_fit.transform(scaled_values)
+    umap_data = pd.DataFrame(umap_data, index=numerical_data.index)
+    # Set UMAP column names with component number
+    new_column_names = ["UMAP_" + str(i) for i in range(1, len(umap_data.columns) + 1)]
+    # Format the output
+    column_mapper = dict(zip(list(umap_data.columns), new_column_names))
+    umap_data = umap_data.rename(columns=column_mapper)
+    output = pd.concat([data_import, umap_data], axis=1)
+    return output, list(categorical_data.columns), new_column_names
+# PCA function for the Sample Selection module
+def pca_maker(data_import):
+    numerical_data, categorical_data, scaled_values = col_cat(data_import)
     # Compute a 6 components PCA on scaled values
     pca = PCA(n_components=6)
     pca_fit = pca.fit(scaled_values)