Fixed bugs due to new echem file format having a column called index. Added support for timestamp column to be called "Data_Time"

Author: be-smith

src/datalab_app_plugin_insitu/apps/xrd/utils.py

@@ -170,13 +170,12 @@ def process_local_xrd_data(
 
                 df_echem = xrd_data["Time_series_data"]["data"]
 
+                # Rename timestamp column to echem_timestamp
+                # Note: Timestamp column is already standardized in process_echem_data()
+                # Note: elapsed_time_seconds and elapsed_time_hours are already calculated in process_echem_data()
                 df_echem["Timestamp"] = pd.to_datetime(df_echem["Timestamp"])
                 df_echem = df_echem.rename(columns={"Timestamp": "echem_timestamp"})
-                time_deltas = df_echem.echem_timestamp - df_echem.echem_timestamp.iloc[0]
-                df_echem["elapsed_time_hours"] = [
-                    delta.total_seconds() / 3600 for delta in time_deltas
-                ]
-                df_echem["elapsed_time_seconds"] = [delta.total_seconds() for delta in time_deltas]
+
                 log_data.rename(columns={"start_time": "xrd_timestamp"}, inplace=True)
                 log_data["xrd_timestamp"] = pd.to_datetime(log_data["xrd_timestamp"])
                 df_merged = pd.merge_asof(
@@ -188,6 +187,7 @@ def process_local_xrd_data(
                 )
 
                 # Adding scan_number to the echem data to be used for the lengend later.
+                # Note: Timestamp column is already standardized to "Timestamp" in process_echem_data()
                 echem_merged = pd.merge_asof(
                     xrd_data["Time_series_data"]["data"],
                     xrd_data["log data"][["xrd_timestamp", "scan_number"]],
@@ -205,6 +205,11 @@ def process_local_xrd_data(
                         "elapsed_time_seconds": "time",
                     }
                 )
+
+                # Remove 'index' column if it exists in the data to avoid conflicts with pandas reset_index()
+                if "index" in df_merged.columns:
+                    df_merged = df_merged.drop(columns=["index"])
+
                 xrd_data["index_df"] = df_merged
 
                 # Create a mapping from file_num to exp_num

src/datalab_app_plugin_insitu/echem_utils.py

@@ -37,9 +37,19 @@ def process_echem_data(echem_folder: Path) -> Dict:
     else:
         raise ValueError(f"Echem folder not found: {echem_folder}")
 
+    # Rename timestamp column to a consistent name
     if "Timestamp" in echem_data.columns:
+        timestamp_col = "Timestamp"
+    elif "Date_Time" in echem_data.columns:
+        echem_data = echem_data.rename(columns={"Date_Time": "Timestamp"})
+        timestamp_col = "Timestamp"
+    else:
+        timestamp_col = None
+
+    if timestamp_col:
         time_deltas = echem_data["Timestamp"] - echem_data["Timestamp"].iloc[0]
         echem_data["elapsed_time_seconds"] = [delta.total_seconds() for delta in time_deltas]
+        echem_data["elapsed_time_hours"] = [delta.total_seconds() / 3600 for delta in time_deltas]
         echem_data["Time"] = echem_data["elapsed_time_seconds"]
 
     min_time = echem_data["Time"].min()

src/datalab_app_plugin_insitu/plotting.py

@@ -562,7 +562,7 @@ def _link_plots(
     echemplot_figure.add_tools(crosshair)
     hover = next((tool for tool in heatmap_figure.tools if isinstance(tool, HoverTool)), None)
     if hover:
-        index_df_source = ColumnDataSource(plot_data["index_df"].reset_index())
+        index_df_source = ColumnDataSource(plot_data["index_df"].reset_index(drop=True))
         hover.callback = CustomJS(
             args=dict(
                 line_source=line_source,