Implemented more multithreaded precautions in precalculations and preprocessing pipeline, added concurrency tests and validations on several datacentric functions, refractored preprocessing pipeline, added visual upgrades to the UI and a clear session and bug report buttons

app.jl

@@ -8,6 +8,7 @@ using Libz
 using PlotlyBase
 using CairoMakie
 using Colors
+using Dates
 using MSI_src # Import the new MSIData library
 using Statistics
 using NaturalSort
@@ -113,7 +114,9 @@ function get_processed_mean_spectrum(spectra::Vector{MutableSpectrum}; num_bins=
 
     # 3. Bin intensities
     for s in spectra
-        for i in eachindex(s.mz)
+        # Use minimum length to avoid bounds errors if arrays are mismatched
+        n_points = min(length(s.mz), length(s.intensity))
+        for i in 1:n_points
             bin_index = trunc(Int, (s.mz[i] - min_mz) * inv_bin_step + 1.0)
             final_index = clamp(bin_index, 1, num_bins)
             intensity_sum[final_index] += s.intensity[i]
@@ -147,7 +150,9 @@ function get_processed_sum_spectrum(spectra::Vector{MutableSpectrum}; num_bins=2
     inv_bin_step = 1.0 / bin_step
 
     for s in spectra
-        for i in eachindex(s.mz)
+        # Use minimum length to avoid bounds errors if arrays are mismatched
+        n_points = min(length(s.mz), length(s.intensity))
+        for i in 1:n_points
             bin_index = trunc(Int, (s.mz[i] - min_mz) * inv_bin_step + 1.0)
             final_index = clamp(bin_index, 1, num_bins)
             intensity_sum[final_index] += s.intensity[i]
@@ -157,11 +162,44 @@ function get_processed_sum_spectrum(spectra::Vector{MutableSpectrum}; num_bins=2
     return collect(mz_bins), intensity_sum
 end
 
+INITIAL_MODEL_STATE = Dict{Symbol,Any}()
+
+# Function to capture initial state (also outside @app block)
+function capture_initial_state!(model)
+    empty!(INITIAL_MODEL_STATE)
+    for name in fieldnames(typeof(model))
+        if !startswith(String(name), "_")
+            INITIAL_MODEL_STATE[name] = deepcopy(getfield(model, name))
+        end
+    end
+    println("Captured $(length(INITIAL_MODEL_STATE)) reactive variables")
+end
+
 @genietools
 
 # == Reactive code ==
+#=
+macro ui_log(message, level="INFO", log_entries)
+    quote
+        local timestamp = Dates.format(now(), "HH:MM:SS")
+        local new_entry = Dict("time" => timestamp, "message" => string($(esc(message))), "level" => $(esc(level)))
+        println("log entries value: $log_entries")
+        pushfirst!(log_entries, new_entry)
+        if length(log_entries) > 100
+            popfirst!(log_entries)
+        end
+        push!(__model__)
+    end
+end
+=#
+
 # Reactive code to make the UI interactive
 @app begin
+    # == Notification & Logs ==
+    # @in log_entries = Dict{String,Any}[]
+    # @in show_log_sidebar = false
+    @in showBugModal = false
+    
     # == Loading Screen Variables ==
     @in is_initializing = true
     @in initialization_message = "Initializing..."
@@ -348,6 +386,7 @@ end
     @in export_params_btn = false # Export parameters to file
     @in import_params_btn = false # Import parameters from file
     @in save_feature_matrix_btn = false # Save feature matrix results
+    @in reset_session_btn = false # Deep session reset
 
     # Preprocessing results
     @in selected_spectrum_id_for_plot = 1
@@ -616,12 +655,64 @@ end
     # Reactive handlers watch a variable and execute a block of code when its value changes
     # The onbutton handler will set the variable to false after the block is executed
 
-    # This handler correctly uses pick_file and loads the selected file
+    @onbutton reset_session_btn begin
-    # as the active dataset for the UI.
+        is_processing = true
+        push!(__model__)
+        
+        try
+            # 1. Clear large data objects explicitly
+            msi_data = nothing
+            feature_matrix_result = nothing
+            bin_info_result = nothing
+            
+            # 2. Reset ALL reactive variables using captured initial state
+            if !isempty(INITIAL_MODEL_STATE)
+                for (name, value) in INITIAL_MODEL_STATE
+                    setfield!(__model__, name, deepcopy(value))
+                end
+                msg = "Session reset: restored $(length(INITIAL_MODEL_STATE)) variables to initial state."
+            else
+                msg = "Warning: No initial state captured. Using partial reset."
+            end
+            
+            # 3. Reset file lists (these will be repopulated by normal operation)
+            msi_bmp = sort(filter(filename -> startswith(filename, "MSI_") && endswith(filename, ".bmp"), readdir("public")), lt=natural)
+            col_msi_png = sort(filter(filename -> startswith(filename, "colorbar_MSI_") && endswith(filename, ".png"), readdir("public")), lt=natural)
+            triq_bmp = sort(filter(filename -> startswith(filename, "TrIQ_") && endswith(filename, ".bmp"), readdir("public")), lt=natural)
+            col_triq_png = sort(filter(filename -> startswith(filename, "colorbar_TrIQ_") && endswith(filename, ".png"), readdir("public")), lt=natural)
+            
+            # 4. Clear any cached images/plots
+            imgInt = "/.bmp"
+            imgIntT = "/.bmp"
+            colorbar = "/.png"
+            colorbarT = "/.png"
+            
+            # 5. Reset plot data to default traces
+            traceImg = PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}())
+            plotdataImg = [traceImg]
+            plotdataImgT = [traceImg]
+            plotdata = [PlotlyBase.scatter(x=Vector{Float64}(), y=Vector{Float64}(), mode="lines")]
+            
+            # 6. Aggressive garbage collection
+            GC.gc(true)
+            if Sys.islinux()
+                ccall(:malloc_trim, Int32, (Int32,), 0)
+            end
+            println("Session reset successfully.")
+        catch e
+            println("Error during session reset: $e")
+            msg = "Reset error: $e"
+        finally
+            is_processing = false
+        end
+    end
+
     @onbutton btnSearch begin
         is_processing = true
+        push!(__model__)
         picked_route = pick_file(; filterlist="imzML,imzml,mzML,mzml")
-        if isempty(picked_route)
+        
+        if isnothing(picked_route) || isempty(picked_route)
             is_processing = false
             return
         end
@@ -938,6 +1029,7 @@ end
 
     @onbutton export_params_btn begin
         is_processing = true
+        push!(__model__)
         params_to_export = Dict(
             "pipeline_step_order" => pipeline_step_order,
             "enable_standards" => enable_standards, # Export global flag
@@ -1011,6 +1103,7 @@ end
 
     @onbutton import_params_btn begin
         is_processing = true
+        push!(__model__)
         picked_file = pick_file(filterlist="json")
         if isempty(picked_file)
             is_processing = false
@@ -1021,39 +1114,57 @@ end
             json_string = read(picked_file, String)
             params = JSON.parse(json_string)
 
-            # This is a list of all known reactive variables that can be imported.
+            # Import special variables first
-            # This prevents arbitrary variable assignment.
+            if haskey(params, "reference_peaks_list")
-            known_params = [
+                reference_peaks_list = params["reference_peaks_list"]
-                "stabilization_method", "smoothing_method", "smoothing_window", "smoothing_order",
-                "baseline_method", "baseline_iterations", "baseline_window", "normalization_method",
-                "alignment_method", "alignment_span", "alignment_tolerance", "alignment_tolerance_unit",
-                "alignment_max_shift_ppm", "alignment_min_matched_peaks", "peak_picking_method",
-                "peak_picking_snr_threshold", "peak_picking_half_window", "peak_picking_min_peak_prominence",
-                "peak_picking_merge_peaks_tolerance", "peak_picking_min_peak_width_ppm",
-                "peak_picking_max_peak_width_ppm", "peak_picking_min_peak_shape_r2", "binning_method",
-                "binning_tolerance", "binning_tolerance_unit", "binning_frequency_threshold",
-                "binning_min_peak_per_bin", "binning_max_bin_width_ppm", "binning_intensity_weighted_centers",
-                "binning_num_uniform_bins", "calibration_fit_order", "calibration_ppm_tolerance",
-                "peak_selection_min_snr", "peak_selection_min_fwhm_ppm", "peak_selection_max_fwhm_ppm",
-                "peak_selection_min_shape_r2", "peak_selection_frequency_threshold", "peak_selection_correlation_threshold"
-            ]
-
-            for (key, value) in params
-                if key == "reference_peaks_list"
-                    reference_peaks_list = value
-                elseif key == "pipeline_step_order"
-                    pipeline_step_order = value
-                elseif key == "enable_standards"
-                    enable_standards = value
-                elseif key in known_params
-                    # Use getfield and setproperty! to update reactive variables by name
-                    if hasfield(typeof(@__MODULE__), Symbol(key))
-                        getfield(@__MODULE__, Symbol(key))[] = value
-                    end
-                else
-                    @warn "Unknown parameter '$key' found in JSON file. Skipping."
-                end
             end
+            if haskey(params, "pipeline_step_order")
+                pipeline_step_order = params["pipeline_step_order"]
+            end
+            if haskey(params, "enable_standards")
+                enable_standards = params["enable_standards"]
+            end
+
+            # Import regular parameters with explicit assignments
+            haskey(params, "stabilization_method") && (stabilization_method = params["stabilization_method"])
+            haskey(params, "smoothing_method") && (smoothing_method = params["smoothing_method"])
+            haskey(params, "smoothing_window") && (smoothing_window = params["smoothing_window"])
+            haskey(params, "smoothing_order") && (smoothing_order = params["smoothing_order"])
+            haskey(params, "baseline_method") && (baseline_method = params["baseline_method"])
+            haskey(params, "baseline_iterations") && (baseline_iterations = params["baseline_iterations"])
+            haskey(params, "baseline_window") && (baseline_window = params["baseline_window"])
+            haskey(params, "normalization_method") && (normalization_method = params["normalization_method"])
+            haskey(params, "alignment_method") && (alignment_method = params["alignment_method"])
+            haskey(params, "alignment_span") && (alignment_span = params["alignment_span"])
+            haskey(params, "alignment_tolerance") && (alignment_tolerance = params["alignment_tolerance"])
+            haskey(params, "alignment_tolerance_unit") && (alignment_tolerance_unit = params["alignment_tolerance_unit"])
+            haskey(params, "alignment_max_shift_ppm") && (alignment_max_shift_ppm = params["alignment_max_shift_ppm"])
+            haskey(params, "alignment_min_matched_peaks") && (alignment_min_matched_peaks = params["alignment_min_matched_peaks"])
+            haskey(params, "peak_picking_method") && (peak_picking_method = params["peak_picking_method"])
+            haskey(params, "peak_picking_snr_threshold") && (peak_picking_snr_threshold = params["peak_picking_snr_threshold"])
+            haskey(params, "peak_picking_half_window") && (peak_picking_half_window = params["peak_picking_half_window"])
+            haskey(params, "peak_picking_min_peak_prominence") && (peak_picking_min_peak_prominence = params["peak_picking_min_peak_prominence"])
+            haskey(params, "peak_picking_merge_peaks_tolerance") && (peak_picking_merge_peaks_tolerance = params["peak_picking_merge_peaks_tolerance"])
+            haskey(params, "peak_picking_min_peak_width_ppm") && (peak_picking_min_peak_width_ppm = params["peak_picking_min_peak_width_ppm"])
+            haskey(params, "peak_picking_max_peak_width_ppm") && (peak_picking_max_peak_width_ppm = params["peak_picking_max_peak_width_ppm"])
+            haskey(params, "peak_picking_min_peak_shape_r2") && (peak_picking_min_peak_shape_r2 = params["peak_picking_min_peak_shape_r2"])
+            haskey(params, "binning_method") && (binning_method = params["binning_method"])
+            haskey(params, "binning_tolerance") && (binning_tolerance = params["binning_tolerance"])
+            haskey(params, "binning_tolerance_unit") && (binning_tolerance_unit = params["binning_tolerance_unit"])
+            haskey(params, "binning_frequency_threshold") && (binning_frequency_threshold = params["binning_frequency_threshold"])
+            haskey(params, "binning_min_peak_per_bin") && (binning_min_peak_per_bin = params["binning_min_peak_per_bin"])
+            haskey(params, "binning_max_bin_width_ppm") && (binning_max_bin_width_ppm = params["binning_max_bin_width_ppm"])
+            haskey(params, "binning_intensity_weighted_centers") && (binning_intensity_weighted_centers = params["binning_intensity_weighted_centers"])
+            haskey(params, "binning_num_uniform_bins") && (binning_num_uniform_bins = params["binning_num_uniform_bins"])
+            haskey(params, "calibration_fit_order") && (calibration_fit_order = params["calibration_fit_order"])
+            haskey(params, "calibration_ppm_tolerance") && (calibration_ppm_tolerance = params["calibration_ppm_tolerance"])
+            haskey(params, "peak_selection_min_snr") && (peak_selection_min_snr = params["peak_selection_min_snr"])
+            haskey(params, "peak_selection_min_fwhm_ppm") && (peak_selection_min_fwhm_ppm = params["peak_selection_min_fwhm_ppm"])
+            haskey(params, "peak_selection_max_fwhm_ppm") && (peak_selection_max_fwhm_ppm = params["peak_selection_max_fwhm_ppm"])
+            haskey(params, "peak_selection_min_shape_r2") && (peak_selection_min_shape_r2 = params["peak_selection_min_shape_r2"])
+            haskey(params, "peak_selection_frequency_threshold") && (peak_selection_frequency_threshold = params["peak_selection_frequency_threshold"])
+            haskey(params, "peak_selection_correlation_threshold") && (peak_selection_correlation_threshold = params["peak_selection_correlation_threshold"])
+
             msg = "Parameters imported successfully from $(basename(picked_file))."
         catch e
             msg = "Failed to import parameters: $e"
@@ -1110,13 +1221,14 @@ end
 
     @onbutton run_full_pipeline begin
         is_processing = true
+        push!(__model__)
+        overall_progress = 0.0
+        local pipeline_msi_data = nothing
+        local current_spectra = Vector{MutableSpectrum}()
         current_pipeline_step = "Initializing..."
-        println("DEBUG: run_full_pipeline started.")
-        # println("DEBUG: Current pipeline_step_order configuration: $pipeline_step_order")
         
         try
             # --- 1. Initial Checks and Data Loading ---
-            println("DEBUG: Performing initial checks and data loading...")
             if isempty(selected_folder_main)
                 msg = "No dataset loaded. Please load a file using 'Select an imzMl / mzML file'."
                 warning_msg = true
@@ -1136,26 +1248,31 @@ end
             target_path = entry["source_path"]
 
             # Ensure msi_data is for the currently selected file and load if needed
-            if msi_data === nothing || full_route != target_path
+            # NOTE: For the pipeline, we will open a DEDICATED instance to avoid race conditions
-                println("DEBUG: Active file path changed or data not in memory. Reloading MSI data: $(basename(target_path))")
+            # with the global msi_data used for plotting/interactive exploration.
-                if msi_data !== nothing; close(msi_data); end
+            println("DEBUG: Opening isolated MSIData instance for pipeline stability...")
-                msg = "Reloading $(basename(target_path)) for analysis..."
+            pipeline_msi_data = OpenMSIData(target_path)
-                full_route = target_path
-                msi_data = OpenMSIData(target_path)
 
-                # Determine plot mode from loaded data
+            # Determine plot mode from metadata for correct visualization late
-                df = msi_data.spectrum_stats_df
+            metadata = pipeline_msi_data.instrument_metadata
+            acq_mode = metadata !== nothing ? metadata.acquisition_mode : :unknown
+            
+            if acq_mode == :centroid
+                last_plot_mode = "stem"
+            elseif acq_mode == :profile
+                last_plot_mode = "lines"
+            else
+                # Fallback to stats if mode is unknown
+                df = pipeline_msi_data.spectrum_stats_df
                 if df !== nothing && "Mode" in names(df)
                     profile_count = count(==(MSI_src.PROFILE), df.Mode)
                     total_count = length(df.Mode)
                     last_plot_mode = profile_count > total_count / 2 ? "lines" : "stem"
-                    println("DEBUG: Auto-detected plot mode for pipeline: $(last_plot_mode)")
                 else
                     last_plot_mode = "lines" # Default
                 end
-            else
-                println("DEBUG: Using already loaded MSI data for $(basename(target_path)).")
             end
+            println("DEBUG: Auto-detected plot mode from metadata: $(last_plot_mode) (acq_mode: $(acq_mode)) [Initial set]")
 
             # Mask path retrieval from registry
             local mask_path_for_pipeline::Union{String, Nothing} = nothing
@@ -1171,6 +1288,7 @@ end
                         warning_msg = true
                         @warn msg
                         println("DEBUG: $msg")
+                        close(pipeline_msi_data) # Important cleanup
                         return
                     end
                 else
@@ -1178,6 +1296,7 @@ end
                     warning_msg = true
                     @warn msg
                     println("DEBUG: $msg")
+                    close(pipeline_msi_data) # Important cleanup
                     return
                 end
             else
@@ -1185,22 +1304,24 @@ end
             end
 
             # Apply mask if enabled to get indices to process
-            spectrum_indices_to_process = collect(1:length(msi_data.spectra_metadata))
+            # Use pipeline_msi_data for consistency
+            spectrum_indices_to_process = collect(1:length(pipeline_msi_data.spectra_metadata))
             if mask_path_for_pipeline !== nothing
                 current_pipeline_step = "Applying mask..."
                 println("DEBUG: Applying mask matrix to filter spectra...")
-                mask_matrix = load_and_prepare_mask(mask_path_for_pipeline, msi_data.image_dims)
+                mask_matrix = load_and_prepare_mask(mask_path_for_pipeline, pipeline_msi_data.image_dims)
-                masked_indices_set = get_masked_spectrum_indices(msi_data, mask_matrix)
+                masked_indices_set = get_masked_spectrum_indices(pipeline_msi_data, mask_matrix)
                 spectrum_indices_to_process = collect(masked_indices_set)
                 if isempty(spectrum_indices_to_process)
                     msg = "No spectra remaining after applying mask. Aborting pipeline."
                     warning_msg = true
                     println("DEBUG: $msg")
+                    close(pipeline_msi_data) # Important cleanup
                     return
                 end
                 println("DEBUG: $(length(spectrum_indices_to_process)) spectra remaining after mask application.")
             else
-                println("DEBUG: No mask applied. Processing all $(length(msi_data.spectra_metadata)) spectra.")
+                println("DEBUG: No mask applied. Processing all $(length(pipeline_msi_data.spectra_metadata)) spectra.")
             end
 
             # Apply subset processing if enabled
@@ -1211,18 +1332,92 @@ end
                 println("DEBUG: Subset processing enabled. Processing first $(length(spectrum_indices_to_process)) of $n_total spectra.")
             end
 
-            # Initialize spectra data structure
+            # --- BOUNDS VALIDATION AND DIAGNOSTIC LOGGING ---
-            current_pipeline_step = "Loading spectra..."
+            # Validate all indices are within bounds before attempting to load
-            println("DEBUG: Loading $(length(spectrum_indices_to_process)) spectra into MutableSpectrum objects...")
+            max_spectra_idx = length(pipeline_msi_data.spectra_metadata)
-            current_spectra = Vector{MutableSpectrum}(undef, length(spectrum_indices_to_process))
+            println("DEBUG: Total spectra in dataset: $max_spectra_idx")
+            println("DEBUG: Number of indices to process: $(length(spectrum_indices_to_process))")
             
-            Threads.@threads for i in 1:length(spectrum_indices_to_process)
+            if !isempty(spectrum_indices_to_process)
-                original_idx = spectrum_indices_to_process[i]
+                min_idx = minimum(spectrum_indices_to_process)
-                mz, intensity = GetSpectrum(msi_data, original_idx) # Fetch mz and intensity for the current spectrum
+                max_idx = maximum(spectrum_indices_to_process)
-                current_spectra[i] = MutableSpectrum(original_idx, copy(Float64.(mz)), copy(Float64.(intensity)), NamedTuple{(:mz, :intensity, :fwhm, :shape_r2, :snr, :prominence), NTuple{6, Float64}}[])
+                println("DEBUG: Spectrum indices range: $min_idx to $max_idx")
+                
+                # Check for invalid indices
+                invalid_indices = filter(idx -> idx < 1 || idx > max_spectra_idx, spectrum_indices_to_process)
+                if !isempty(invalid_indices)
+                    n_invalid = length(invalid_indices)
+                    sample_invalid = first(sort(invalid_indices), min(10, n_invalid))
+                    msg = "Invalid spectrum indices detected: $n_invalid indices out of range [1, $max_spectra_idx]. First few invalid indices: $sample_invalid"
+                    warning_msg = true
+                    @error msg
+                    println("DEBUG: $msg")
+                    close(pipeline_msi_data) # Important cleanup
+                    return
+                end
+                println("DEBUG: All spectrum indices are valid (within [1, $max_spectra_idx]).")
+            else
+                println("DEBUG: Warning - spectrum_indices_to_process is empty!")
+            end
+            
+            # CRITICAL: Verify indices are unique to prevent race conditions during loading
+            if length(Set(spectrum_indices_to_process)) != length(spectrum_indices_to_process)
+                @warn "Non-unique indices detected in spectrum_indices_to_process. This may cause issues during parallel loading."
+            end
+
+            # Use pipeline_msi_data for reading
+            # Split loading into chunks to update progress bar
+            n_spectra = length(spectrum_indices_to_process)
+            println("DEBUG: Loading $n_spectra spectra into MutableSpectrum objects...")
+            current_spectra = Vector{MutableSpectrum}(undef, n_spectra)
+            
+            chunk_size = max(1, n_spectra ÷ 10) # Update progress every 10%
+            for chunk_start in 1:chunk_size:n_spectra
+                chunk_end = min(chunk_start + chunk_size - 1, n_spectra)
+                
+                Threads.@threads for i in chunk_start:chunk_end
+                    local original_idx = spectrum_indices_to_process[i]
+                    local mz, intensity # Enforce thread-local scope
+                    
+                    try
+                        mz, intensity = GetSpectrum(pipeline_msi_data, original_idx)
+                        
+                        # Diagnostic check for length mismatch and defensive truncation
+                        l_mz = length(mz)
+                        l_int = length(intensity)
+                        if l_mz != l_int
+                            new_len = min(l_mz, l_int)
+                            @warn "CRITICAL: Mismatch during loading at index $original_idx. mz=$l_mz, int=$l_int. TRUNCATING."
+                            mz = mz[1:new_len]
+                            intensity = intensity[1:new_len]
+                        end
+
+                        current_spectra[i] = MutableSpectrum(original_idx, copy(Float64.(mz)), copy(Float64.(intensity)), NamedTuple{(:mz, :intensity, :fwhm, :shape_r2, :snr, :prominence), NTuple{6, Float64}}[])
+                    catch loop_error
+                        rethrow(loop_error)
+                    end
+                end
+                
+                overall_progress = (chunk_end / n_spectra) * 0.2 # Loading is first 20%
+                push!(__model__)
             end
             println("DEBUG: All spectra loaded into temporary structure for processing.")
             
+            # We can now close the local MSI data instance as we have loaded everything into memory
+            # However, if we want to support lazy loading scenarios later, we might keep it open.
+            # For now, let's close it here to free up file handles early, 
+            # UNLESS `execute_full_preprocessing` needs it (it doesn't seem to based on signature).
+            close(pipeline_msi_data) 
+            pipeline_msi_data = nothing # Prevent accidental use
+            
+            # Aggressive memory cleanup to return memory to OS
+            println("DEBUG: Performing aggressive memory cleanup...")
+            GC.gc(true)  # Full garbage collection with all generations
+            if Sys.islinux()
+                ccall(:malloc_trim, Int32, (Int32,), 0)
+            end
+            println("DEBUG: Closed local pipeline MSIData instance and freed memory.")
+
 
             # --- 2. Parameter Assembly with Validation ---
             current_pipeline_step = "Configuring parameters..."
@@ -1450,8 +1645,16 @@ end
                 mask_path_for_pipeline
             ) do step
                 current_pipeline_step = "Processing: $step"
-                println("DEBUG: Processing step: $step")
+                
+                # Update progress based on step index
+                step_idx = findfirst(==(step), pipeline_stp)
+                if step_idx !== nothing
+                    # Preprocessing is 20% to 90% (total 70%)
+                    overall_progress = 0.2 + (step_idx / length(pipeline_stp)) * 0.7
+                end
+                push!(__model__)
             end
+
             println("DEBUG: Pipeline execution finished.")
 
             # 4. Update Results Display
@@ -1638,7 +1841,8 @@ end
                 mkpath(output_dir)
                 save_feature_matrix(feature_matrix_result, bin_info_result, output_dir)
                 msg = "Pipeline completed successfully. Feature matrix saved."
-                println("DEBUG: Feature matrix saved to $output_dir")
+                overall_progress = 1.0
+                push!(__model__)
             else
                 msg = "Pipeline completed successfully. No feature matrix generated (binning step not enabled)."
                 println("DEBUG: $msg")
@@ -1650,18 +1854,71 @@ end
             @error "Pipeline failed" exception=(e, catch_backtrace())
             println("DEBUG: Pipeline caught an exception: $e")
         finally
+            # Aggressive memory cleanup
+            println("DEBUG: Starting aggressive memory cleanup...")
+            
+            # Explicitly clear large data structures
+            try
+                if current_spectra !== nothing && !isempty(current_spectra)
+                    # Deep clear individual objects to break references effectively
+                    # Use isassigned to prevent UndefRefError if loading failed halfway
+                    for i in eachindex(current_spectra)
+                        if isassigned(current_spectra, i)
+                            s = current_spectra[i]
+                            s.mz = Float64[]
+                            s.intensity = Float64[]
+                            empty!(s.peaks)
+                        end
+                    end
+                    empty!(current_spectra)
+                end
+                current_spectra = nothing
+                
+                if feature_matrix_result !== nothing
+                    feature_matrix_result = nothing
+                end
+                
+                # Close any open pipeline data handles
+                if pipeline_msi_data !== nothing
+                    try
+                        close(pipeline_msi_data)
+                    catch
+                        # Already closed, ignore
+                    end
+                    pipeline_msi_data = nothing
+                end
+                
+                println("DEBUG: Data structures cleared. Triggering garbage collection...")
+            catch cleanup_error
+                @warn "Error during data cleanup: $cleanup_error"
+            end
+            
             is_processing = false
+            overall_progress = 0.0
             current_pipeline_step = ""
             println("DEBUG: run_full_pipeline finished (finally block).")
-            GC.gc() # Trigger garbage collection
+            
+            # Force garbage collection multiple times for thorough cleanup
+            GC.gc()
+            GC.gc()  # Second pass to catch any circular references
+            
+            # On Linux/Unix, force Julia to return memory to OS
             if Sys.islinux()
-                ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure Julia returns the freed memory to OS
+                try
+                    ccall(:malloc_trim, Int32, (Int32,), 0)
+                    println("DEBUG: malloc_trim called successfully (Linux).")
+                catch e
+                    @warn "malloc_trim failed: $e"
+                end
             end
+            
+            println("DEBUG: Memory cleanup complete.")
         end
     end
 
     @onbutton recalculate_suggestions_btn begin
         is_processing = true
+        push!(__model__)
         if msi_data === nothing
             msg = "Please load a file first."
             warning_msg = true
@@ -1903,6 +2160,7 @@ end
     # This new handler correctly adds the file from full_route to the batch list.
     @onbutton btnAddBatch begin
         is_processing = true
+        push!(__model__)
         if isempty(full_route) || full_route == "unknown (manually added)"
             msg = "No active file selected to add to batch."
             warning_msg = true
@@ -1923,6 +2181,7 @@ end
 
     @onbutton clear_batch_btn begin
         is_processing = true
+        push!(__model__)
         selected_files = String[]
         batch_file_count = 0
         msg = "Batch cleared"
@@ -1974,6 +2233,7 @@ end
 
     @onchange btnSearchMzml, btnSearchSync begin
         is_processing = true
+        push!(__model__)
         if btnSearchMzml
             picked_route = pick_file(; filterlist="mzML,mzml")
             if !isempty(picked_route)
@@ -1997,6 +2257,7 @@ end
 
     @onbutton convert_process begin
         is_processing = true
+        push!(__model__)
         if isempty(mzml_full_route) || isempty(sync_full_route)
             msg_conversion = "Please select both an .mzML file and a .txt sync file."
             warning_msg = true
@@ -2029,6 +2290,7 @@ end
             @error "Conversion failed" exception=(e, catch_backtrace())
         finally
             is_processing = false
+            overall_progress = 0.0
             # Re-enable button if files are still selected
             btnConvertDisable = isempty(mzml_full_route) || isempty(sync_full_route)
         end
@@ -2061,6 +2323,7 @@ end
                 return
             end
             is_processing = true
+            push!(__model__)
 
             masses = Float64[]
             try
@@ -2087,7 +2350,8 @@ end
 
             for (file_idx, file_path) in enumerate(current_selected_files)
                 progress_message = "Processing file $(file_idx)/$(num_files): $(basename(file_path))"
-                overall_progress = current_step / total_steps
+                overall_progress = (file_idx - 1) / num_files
+                push!(__model__)
                 
                 all_params = (
                     tolerance = current_tol,
@@ -2218,6 +2482,10 @@ end
     end
 
     @onbutton createMeanPlot @time begin
+        # Pre-initialize for safe cleanup in finally block
+        local xSpectraMz = Vector{Float64}()
+        local ySpectraMz = Vector{Float64}()
+        
         if isempty(selected_folder_main)
             msg = "No dataset selected. Please process a file and select a folder first."
             warning_msg = true
@@ -2225,6 +2493,7 @@ end
         end
 
         is_processing = true
+        push!(__model__)
 
         try
             sTime = time()
@@ -2275,12 +2544,25 @@ end
             msg = "Plot loaded in $(eTime) seconds"
             log_memory_usage("Mean Plot Generated", msi_data)
         catch e
-                    msg = "Could not generate mean spectrum plot: $e"
+            msg = "Could not generate mean spectrum plot: $e"
-                    warning_msg = true
+            warning_msg = true
-                    @error "Mean spectrum plotting failed" exception=(e, catch_backtrace())
+            @error "Mean spectrum plotting failed" exception=(e, catch_backtrace())
-                finally
+        finally
-                    is_processing = false
+            is_processing = false
-                    GC.gc() # Trigger garbage collection
+            try
+                if plotdata_before !== nothing
+                    plotdata_before = nothing
+                end
+                if !isempty(xSpectraMz)
+                    empty!(xSpectraMz)
+                end
+                if !isempty(ySpectraMz)
+                    empty!(ySpectraMz)
+                end
+            catch
+            end
+            
+            GC.gc() # Trigger garbage collection
             if Sys.islinux()
                 ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure Julia returns the freed memory to OS
             end
@@ -2288,6 +2570,10 @@ end
     end
 
     @onbutton createSumPlot @time begin
+        # Pre-initialize for safe cleanup in finally block
+        local xSpectraMz = Vector{Float64}()
+        local ySpectraMz = Vector{Float64}()
+
         if isempty(selected_folder_main)
             msg = "No dataset selected. Please process a file and select a folder first."
             warning_msg = true
@@ -2295,6 +2581,7 @@ end
         end
 
         is_processing = true
+        push!(__model__)
         msg = "Loading total spectrum plot for $(selected_folder_main)..."
         
         try
@@ -2350,6 +2637,18 @@ end
             @error "Total spectrum plotting failed" exception=(e, catch_backtrace())
         finally
             is_processing = false
+            try
+                if plotdata_before !== nothing
+                    plotdata_before = nothing
+                end
+                if !isempty(xSpectraMz)
+                    empty!(xSpectraMz)
+                end
+                if !isempty(ySpectraMz)
+                    empty!(ySpectraMz)
+                end
+            catch
+            end
             GC.gc() # Trigger garbage collection
             if Sys.islinux()
                 ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure Julia returns the freed memory to OS
@@ -2365,6 +2664,7 @@ end
         end
 
         is_processing = true
+        push!(__model__)
         msg = "Loading plot for $(selected_folder_main)..."
 
         try
@@ -2479,6 +2779,7 @@ end
         end
 
         is_processing = true
+        push!(__model__)
         msg = "Loading plot for $(selected_folder_main)..."
 
         try
@@ -3022,6 +3323,7 @@ end
             return
         end
         is_processing = true
+        push!(__model__)
 
         try
             # --- Get Mask Path ---
@@ -3079,6 +3381,7 @@ end
         end
 
         is_processing = true
+        push!(__model__)
 
         try
             # --- Get Mask Path ---
@@ -3137,6 +3440,7 @@ end
         end
 
         is_processing = true
+        push!(__model__)
         
         try
             sTime=time()
@@ -3175,6 +3479,7 @@ end
         end
 
         is_processing = true
+        push!(__model__)
         
         try
             sTime=time()
@@ -3401,6 +3706,10 @@ end
     @mounted watchplots()
 
     @onchange isready begin
+        # Capture state on first run only
+        if isempty(INITIAL_MODEL_STATE)
+            capture_initial_state!(__model__)
+        end
         # is_processing = true
         if isready && !registry_init_done
             sTime=time()
@@ -3471,6 +3780,5 @@ end
     end
 end
 # == Pages ==
-# Register a new route and the page that will be loaded on access
 @page("/", "app.jl.html")
 end

julia_imzML_visual.jl

@@ -1,6 +1,5 @@
 # julia_imzML_visual.jl
 
-const REGISTRY_LOCK = ReentrantLock()
 
 """
     increment_image(current_image, image_list)

mask.jl

@@ -9,7 +9,7 @@ using Statistics, NaturalSort, LinearAlgebra, StipplePlotly
 using Base.Filesystem: mv
 
 using MSI_src
-using .MSI_src: MSIData, process_image_pipeline
+using .MSI_src: MSIData, process_image_pipeline, REGISTRY_LOCK
 
 # Plot Handling
 include("./julia_imzML_visual.jl")
@@ -799,8 +799,10 @@ end
                 end
                 
                 # Save the updated registry
-                open(reg_path, "w") do f
+                lock(REGISTRY_LOCK) do
-                    JSON.print(f, registry, 4)
+                    open(reg_path, "w") do f
+                        JSON.print(f, registry, 4)
+                    end
                 end
                 
                 @info "Registry updated with mask: $(final_mask_name)"
@@ -874,8 +876,10 @@ end
 
                 if !isempty(new_folders) || !isempty(removed_folders)
                     println("Registry changed, saving...")
-                    open(reg_path, "w") do f
+                    lock(REGISTRY_LOCK) do
-                        JSON.print(f, registry, 4)
+                        open(reg_path, "w") do f
+                            JSON.print(f, registry, 4)
+                        end
                     end
                 end
                 

src/Common.jl

@@ -82,30 +82,35 @@ A dramatically simplified buffer pool that avoids complex locking.
 mutable struct SimpleBufferPool
     buffers::Dict{Int, Vector{Vector{UInt8}}}
     max_pool_size::Int
+    lock::ReentrantLock  # Add lock for thread safety
 end
 
-SimpleBufferPool() = SimpleBufferPool(Dict{Int, Vector{Vector{UInt8}}}(), 50)
+SimpleBufferPool() = SimpleBufferPool(Dict{Int, Vector{Vector{UInt8}}}(), 50, ReentrantLock())
 
 function get_buffer!(pool::SimpleBufferPool, size::Int)::Vector{UInt8}
-    # Check for existing buffers of exact size first
+    lock(pool.lock) do
-    if haskey(pool.buffers, size) && !isempty(pool.buffers[size])
+        # Check for existing buffers of exact size first
-        return pop!(pool.buffers[size])
+        if haskey(pool.buffers, size) && !isempty(pool.buffers[size])
+            return pop!(pool.buffers[size])
+        end
     end
     
-    # No suitable buffer found, allocate new one
+    # No suitable buffer found, allocate new one (outside lock to reduce contention)
     return Vector{UInt8}(undef, size)
 end
 
 function release_buffer!(pool::SimpleBufferPool, buffer::Vector{UInt8})
     size = length(buffer)
     
-    if !haskey(pool.buffers, size)
+    lock(pool.lock) do
-        pool.buffers[size] = Vector{Vector{UInt8}}()
+        if !haskey(pool.buffers, size)
-    end
+            pool.buffers[size] = Vector{Vector{UInt8}}()
+        end
         
-    # Limit pool size to prevent memory bloat
+        # Limit pool size to prevent memory bloat
-    if length(pool.buffers[size]) < pool.max_pool_size
+        if length(pool.buffers[size]) < pool.max_pool_size
-        push!(pool.buffers[size], buffer)
+            push!(pool.buffers[size], buffer)
+        end
     end
     # If pool is full, let buffer get GC'd
 end

src/MSIData.jl

@@ -35,7 +35,12 @@ Atomically seeks to a position and reads data into an array. This is the thread-
 way to read from a specific offset in the file.
 """
 function read_at!(tsfh::ThreadSafeFileHandle, a::AbstractArray, pos::Integer)
+    if pos < 0
+        throw(ArgumentError("Invalid seek position: $pos"))
+    end
     lock(tsfh.lock) do
+        # Also check against file size if possible, though filesize() might be expensive to call repeatedly
+        # Let's rely on seek throwing if it goes way out of bounds, but catch the negative case which is definitely an error.
         seek(tsfh.handle, pos)
         read!(tsfh.handle, a)
     end
@@ -510,6 +515,19 @@ function read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata)
     mz = Array{source.mz_format}(undef, meta.mz_asset.encoded_length)
     intensity = Array{source.intensity_format}(undef, meta.int_asset.encoded_length)
 
+    # Validate offsets before reading
+    file_size = filesize(source.ibd_handle)
+    
+    mz_end = meta.mz_asset.offset + sizeof(source.mz_format) * meta.mz_asset.encoded_length
+    if meta.mz_asset.offset < 0 || mz_end > file_size
+        throw(FileFormatError("Invalid m/z data offset/length for spectrum $(meta.id): offset=$(meta.mz_asset.offset), end=$mz_end, file_size=$file_size"))
+    end
+
+    int_end = meta.int_asset.offset + sizeof(source.intensity_format) * meta.int_asset.encoded_length
+    if meta.int_asset.offset < 0 || int_end > file_size
+        throw(FileFormatError("Invalid intensity data offset/length for spectrum $(meta.id): offset=$(meta.int_asset.offset), end=$int_end, file_size=$file_size"))
+    end
+
     # Use the new atomic read_at! method for thread-safety
     read_at!(source.ibd_handle, mz, meta.mz_asset.offset)
     read_at!(source.ibd_handle, intensity, meta.int_asset.offset)

src/MSI_src.jl

@@ -19,7 +19,11 @@ export OpenMSIData,
        get_global_mz_range,
        MSIData,
        _iterate_spectra_fast,
-       validate_spectrum
+       validate_spectrum,
+       REGISTRY_LOCK
+
+# Define shared registry lock
+const REGISTRY_LOCK = ReentrantLock()
 
 # Export the public preprocessing & precalculations API
 export run_preprocessing_analysis,

src/Precalculations.jl

@@ -1,4 +1,5 @@
 using StatsBase # For mean, std, median, quantile, mad
+using Base.Threads # For atomic counters and locking
 
 # =============================================================================
 # 7) Spatial & Advanced Processing (Stubs & New Functions)
@@ -80,11 +81,12 @@ function analyze_mass_accuracy(
     println("Analyzing mass accuracy for $(length(spectrum_indices)) spectra...")
 
     all_ppm_errors = Float64[]
-    total_matched_peaks = 0
+    total_matched_peaks = Atomic{Int}(0)
-    total_spectra_processed = 0
+    total_spectra_processed = Atomic{Int}(0)
+    results_lock = ReentrantLock()
 
     _iterate_spectra_fast(msi_data, spectrum_indices) do idx, mz, intensity
-        total_spectra_processed += 1
+        atomic_add!(total_spectra_processed, 1)
         if !validate_spectrum(mz, intensity)
             @warn "Spectrum $idx is invalid, skipping mass accuracy analysis for it."
             return
@@ -106,8 +108,10 @@ function analyze_mass_accuracy(
             end
 
             if best_matched_peak_mz !== nothing
-                push!(all_ppm_errors, min_ppm_error)
+                lock(results_lock) do
-                total_matched_peaks += 1
+                    push!(all_ppm_errors, min_ppm_error)
+                end
+                atomic_add!(total_matched_peaks, 1)
             end
         end
     end
@@ -140,8 +144,8 @@ function analyze_mass_accuracy(
         std_ppm_error = std_err,
         min_ppm_error = min_err,
         max_ppm_error = max_err,
-        total_matched_peaks = total_matched_peaks,
+        total_matched_peaks = total_matched_peaks[],
-        total_spectra_analyzed = total_spectra_processed,
+        total_spectra_analyzed = total_spectra_processed[],
         ppm_error_distribution = all_ppm_errors
     )
 end
@@ -318,21 +322,29 @@ function analyze_instrument_characteristics(msi_data::MSIData; sample_indices::A
         return results
     end
     
+    results_lock = ReentrantLock()
+    
     _iterate_spectra_fast(msi_data, sample_indices) do idx, mz, intensity
         # Record spectrum mode
-        push!(spectrum_modes, msi_data.spectra_metadata[idx].mode)
+        lock(results_lock) do
+            push!(spectrum_modes, msi_data.spectra_metadata[idx].mode)
+        end
         
         # Calculate m/z step statistics (for profile data)
         if length(mz) > 1 && msi_data.spectra_metadata[idx].mode == PROFILE
             steps = diff(mz)
             if !isempty(steps)
-                push!(mz_step_sizes, mean(steps))
+                lock(results_lock) do
+                    push!(mz_step_sizes, mean(steps))
+                end
             end
         end
         
         # Record intensity range
         if !isempty(intensity)
-            push!(intensity_ranges, (minimum(intensity), maximum(intensity)))
+            lock(results_lock) do
+                push!(intensity_ranges, (minimum(intensity), maximum(intensity)))
+            end
         end
     end
     
@@ -442,11 +454,15 @@ function analyze_signal_quality(msi_data::MSIData; sample_indices::AbstractVecto
         return results
     end
     
+    results_lock = ReentrantLock()
+
     _iterate_spectra_fast(msi_data, sample_indices) do idx, mz, intensity
         if !isempty(intensity)
             # Noise estimation using MAD
             noise = mad(intensity, normalize=true)
-            push!(noise_levels, noise)
+            lock(results_lock) do
+                push!(noise_levels, noise)
+            end
             
             # FIX: More robust SNR calculation
             valid_intensity = intensity[intensity .> 0]  # Remove zeros
@@ -458,12 +474,16 @@ function analyze_signal_quality(msi_data::MSIData; sample_indices::AbstractVecto
                 if noise_robust > 0 && isfinite(signal_estimate)
                     snr_val = signal_estimate / noise_robust
                     # Cap unrealistic SNR values
-                    push!(snr_distribution, min(snr_val, 1e6))
+                    lock(results_lock) do
+                        push!(snr_distribution, min(snr_val, 1e6))
+                    end
                 end
             end
             
             # Total ion count
-            push!(tic_values, sum(intensity))
+            lock(results_lock) do
+                push!(tic_values, sum(intensity))
+            end
         end
     end
     
@@ -651,20 +671,23 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
         peak_counts = Int[]
         fwhm_values = Float64[]
         
-        spectra_analyzed = 0
+        spectra_analyzed = Atomic{Int}(0)
-        peaks_analyzed = 0
+        peaks_analyzed = Atomic{Int}(0)
+        results_lock = ReentrantLock()
         
         _iterate_spectra_fast(msi_data, spectrum_indices) do idx, mz, intensity
             if length(mz) < 10  # Skip spectra with too few points
                 return
             end
             
-            spectra_analyzed += 1
+            atomic_add!(spectra_analyzed, 1)
             meta = msi_data.spectra_metadata[idx]
             
             # Detect peaks with lower SNR threshold to find more peaks
             peaks = detect_peaks_profile_core(mz, intensity; snr_threshold=2.0)
-            push!(peak_counts, length(peaks))
+            lock(results_lock) do
+                push!(peak_counts, length(peaks))
+            end
             
             if !isempty(peaks)
                 # Analyze the strongest 3 peaks per spectrum
@@ -679,15 +702,17 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
                         if !isnan(fwhm_delta_m) && fwhm_delta_m > 0.001 && fwhm_delta_m < 0.5  # Reasonable range in Da
                             fwhm_ppm = 1e6 * fwhm_delta_m / peak.mz
                             if 5.0 < fwhm_ppm < 500.0  # Reasonable ppm range
-                                push!(peak_widths_ppm, fwhm_ppm)
+                                lock(results_lock) do
-                                push!(fwhm_values, fwhm_delta_m)
+                                    push!(peak_widths_ppm, fwhm_ppm)
+                                    push!(fwhm_values, fwhm_delta_m)
                                     
-                                r2 = _fit_gaussian_and_r2(mz, intensity, peak_idx, 5)
+                                    r2 = _fit_gaussian_and_r2(mz, intensity, peak_idx, 5)
-                                push!(r_squared_values, r2)
+                                    push!(r_squared_values, r2)
-                                peaks_analyzed += 1
+                                end
+                                atomic_add!(peaks_analyzed, 1)
                                 
-                                if peaks_analyzed <= 3
+                                if peaks_analyzed[] <= 3
-                                    println("DEBUG: Peak at m/z $(peak.mz), FWHM = $(fwhm_ppm) ppm, R² = $r2")
+                                    println("DEBUG: Peak at m/z $(peak.mz), FWHM = $(fwhm_ppm) ppm, R² = $(r_squared_values[end])")
                                 end
                             end
                         end
@@ -698,7 +723,7 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
             end
         end
         
-        println("DEBUG: Analyzed $peaks_analyzed peaks from $spectra_analyzed spectra")
+        println("DEBUG: Analyzed $(peaks_analyzed[]) peaks from $(spectra_analyzed[]) spectra")
         
         if !isempty(peak_widths_ppm)
             results[:mean_fwhm_ppm] = mean(peak_widths_ppm)
@@ -727,9 +752,13 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
         
         peak_counts = Int[]
         
+        results_lock = ReentrantLock()
+
         _iterate_spectra_fast(msi_data, spectrum_indices) do idx, mz, intensity
             if !isempty(mz)
-                push!(peak_counts, length(mz))
+                lock(results_lock) do
+                    push!(peak_counts, length(mz))
+                end
             end
         end
         

src/PreprocessingPipeline.jl

@@ -33,7 +33,15 @@ function apply_baseline_correction(spectra::Vector{MutableSpectrum}, params::Dic
 
     Threads.@threads for s in spectra
         if validate_spectrum(s.mz, s.intensity)
+            original_length = length(s.intensity)
             baseline = apply_baseline_correction_core(s.intensity; method=method, iterations=iterations, window=window)
+            
+            # CRITICAL: Ensure baseline has the same length as intensity
+            if length(baseline) != original_length
+                @warn "Baseline correction: length mismatch for spectrum $(s.id). baseline=$(length(baseline)), intensity=$original_length. Skipping this spectrum."
+                continue
+            end
+            
             s.intensity = max.(0.0, s.intensity .- baseline)
         end
     end
@@ -53,7 +61,16 @@ function apply_intensity_transformation(spectra::Vector{MutableSpectrum}, params
 
     Threads.@threads for s in spectra
         if validate_spectrum(s.mz, s.intensity)
-            s.intensity = transform_intensity_core(s.intensity; method=method)
+            original_length = length(s.intensity)
+            transformed = transform_intensity_core(s.intensity; method=method)
+            
+            # CRITICAL: Ensure transformation preserves array length
+            if length(transformed) != original_length
+                @warn "Intensity transformation: length mismatch for spectrum $(s.id). transformed=$(length(transformed)), original=$original_length. Skipping this spectrum."
+                continue
+            end
+            
+            s.intensity = transformed
         end
     end
 end
@@ -76,8 +93,16 @@ function apply_smoothing(spectra::Vector{MutableSpectrum}, params::Dict)
 
     Threads.@threads for s in spectra
         if validate_spectrum(s.mz, s.intensity)
-            smoothed_intensity = max.(0.0, smooth_spectrum_core(s.intensity; method=method, window=window, order=order))
+            original_length = length(s.intensity)
-            s.intensity = smoothed_intensity
+            smoothed_intensity = smooth_spectrum_core(s.intensity; method=method, window=window, order=order)
+            
+            # CRITICAL: Ensure smoothing preserves array length  
+            if length(smoothed_intensity) != original_length
+                @warn "Smoothing: length mismatch for spectrum $(s.id). smoothed=$(length(smoothed_intensity)), original=$original_length, method=$method, window=$window. Skipping this spectrum."
+                continue
+            end
+            
+            s.intensity = max.(0.0, smoothed_intensity)
         end
     end
 end
@@ -104,6 +129,7 @@ function apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
 
     Threads.@threads for s in spectra
         if validate_spectrum(s.mz, s.intensity)
+            # old_len = length(s.peaks)
             if method == :profile
                 s.peaks = detect_peaks_profile_core(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window, min_peak_prominence=min_peak_prominence, merge_peaks_tolerance=merge_peaks_tolerance)
             elseif method == :wavelet
@@ -113,6 +139,7 @@ function apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
             else
                 s.peaks = detect_peaks_profile_core(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
             end
+            # @info "Spectrum $(s.id): detected $(length(s.peaks)) peaks"
         else
             s.peaks = []
         end
@@ -182,12 +209,21 @@ function apply_calibration(spectra::Vector{MutableSpectrum}, params::Dict, refer
     Threads.@threads for i in 1:length(spectra)
         s = spectra[i]
         if validate_spectrum(s.mz, s.intensity)
+            original_length = length(s.mz)
             matched_peaks = find_calibration_peaks_core(s.mz, s.intensity, reference_masses; ppm_tolerance=ppm_tolerance)
             if length(matched_peaks) >= 2
                 measured = sort(collect(values(matched_peaks)))
                 theoretical = sort(collect(keys(matched_peaks)))
                 itp = linear_interpolation(measured, theoretical, extrapolation_bc=Line())
-                s.mz = itp(s.mz) # Modify mz-axis in-place
+                new_mz = itp(s.mz)
+                
+                # CRITICAL: Ensure m/z axis preserves array length
+                if length(new_mz) != original_length
+                    @warn "Calibration: length mismatch for spectrum $(s.id). new_mz=$(length(new_mz)), original=$original_length. Skipping this spectrum."
+                    continue
+                end
+                
+                s.mz = new_mz # Modify mz-axis in-place
             else
                 @warn "Spectrum $(s.id): insufficient reference peaks ($(length(matched_peaks)) found), skipping calibration."
             end
@@ -233,7 +269,16 @@ function apply_peak_alignment(spectra::Vector{MutableSpectrum}, params::Dict)
         current_peaks_mz = [p.mz for p in s.peaks]
         alignment_func = align_peaks_lowess_core(ref_peaks_mz, current_peaks_mz; method=method, tolerance=tolerance, tolerance_unit=tolerance_unit)
         
-        s.mz = alignment_func.(s.mz) # Update m/z axis
+        original_length = length(s.mz)
+        new_mz = alignment_func.(s.mz)
+        
+        # CRITICAL: Ensure alignment preserves array length
+        if length(new_mz) != original_length
+            @warn "Peak alignment: m/z length mismatch for spectrum $(s.id). new_mz=$(length(new_mz)), original=$original_length. Skipping this spectrum."
+            continue
+        end
+        
+        s.mz = new_mz # Update m/z axis
         
         # Update peak m/z values
         for i in 1:length(s.peaks)
@@ -258,7 +303,16 @@ function apply_normalization(spectra::Vector{MutableSpectrum}, params::Dict)
 
     Threads.@threads for s in spectra
         if validate_spectrum(s.mz, s.intensity)
-            s.intensity = apply_normalization_core(s.intensity; method=method)
+            original_length = length(s.intensity)
+            normalized = apply_normalization_core(s.intensity; method=method)
+            
+            # CRITICAL: Ensure normalization preserves array length
+            if length(normalized) != original_length
+                @warn "Normalization: length mismatch for spectrum $(s.id). normalized=$(length(normalized)), original=$original_length. Skipping this spectrum."
+                continue
+            end
+            
+            s.intensity = normalized
         end
     end
 end

start_MSI_GUI.jl

@@ -6,17 +6,14 @@ ENV["GENIE_ENV"] = "dev"
 
 manifest_path = joinpath(@__DIR__, "Manifest.toml")
 
-# Only instantiate in development mode
+# Selective instantiation for faster startup
-if get(ENV, "GENIE_ENV", "dev") != "prod" || !isfile(manifest_path)
+if get(ENV, "GENIE_ENV", "dev") != "prod" && !isfile(manifest_path)
-    @info "Development environment detected. Instantiating packages..."
+    @info "Development environment detected and Manifest.toml missing. Instantiating packages..."
-    
-    if !isfile(manifest_path)
-        @info "Manifest.toml not found. Generating it based on Project.toml..."
-    end
-
     Pkg.resolve()
     Pkg.instantiate()
     Pkg.gc()
+elseif get(ENV, "GENIE_ENV", "dev") != "prod"
+    @info "Manifest.toml found. Skipping Pkg.instantiate() for faster boot. Delete Manifest.toml if you need to re-instantiate."
 end
 
 using Genie

test/test_buffer_pool_concurrency.jl

@@ -0,0 +1,32 @@
+using MSI_src
+using Test
+using Base.Threads
+
+@testset "SimpleBufferPool Concurrency Stress Test" begin
+    pool = MSI_src.SimpleBufferPool()
+    n_threads = Threads.nthreads()
+    n_iterations = 1000
+    buffer_size = 1024
+
+    println("Running buffer pool stress test with $n_threads threads...")
+
+    # Parallel stress test
+    Threads.@threads for i in 1:(n_threads * n_iterations)
+        # get_buffer! and release_buffer! are now thread-safe
+        buf = MSI_src.get_buffer!(pool, buffer_size)
+        
+        # Simulate some work
+        fill!(buf, UInt8(i % 256))
+        
+        MSI_src.release_buffer!(pool, buf)
+    end
+
+    # After stress test, the dictionary should be coherent
+    sizes = collect(keys(pool.buffers))
+    if !isempty(sizes)
+        @test buffer_size ∈ sizes
+        @test length(pool.buffers[buffer_size]) <= pool.max_pool_size
+    end
+    
+    println("Buffer pool stress test PASSED.")
+end

test/test_pipeline_concurrency.jl

@@ -0,0 +1,146 @@
+
+using Pkg
+Pkg.activate(joinpath(@__DIR__, ".."))
+using MSI_src
+using Test
+using Base.Threads
+
+# --- Test Configuration ---
+const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Stomach/Stomach_DHB_uncompressed.imzML"
+
+println("Starting concurrency test with file: $TEST_FILE")
+
+if !isfile(TEST_FILE)
+    error("Test file not found: $TEST_FILE")
+end
+
+# --- Simulation of the Concurrency Issue ---
+
+# Global "UI State" variable simulates the app's msi_data
+global_msi_data = nothing
+const GLOBAL_LOCK = ReentrantLock()
+
+function ui_load_file(path)
+    global global_msi_data
+    lock(GLOBAL_LOCK) do
+        if global_msi_data !== nothing
+            close(global_msi_data)
+        end
+        println("UI: Loading file...")
+        global_msi_data = OpenMSIData(path)
+        println("UI: File loaded.")
+    end
+end
+
+function ui_close_file()
+    global global_msi_data
+    lock(GLOBAL_LOCK) do
+        if global_msi_data !== nothing
+            println("UI: Closing file...")
+            close(global_msi_data)
+            global_msi_data = nothing
+            println("UI: File closed.")
+        end
+    end
+end
+
+# Mock pipeline function that mimics `run_full_pipeline` in app.jl
+# Crucially, it uses its own local `pipeline_msi_data` as per the fix
+function run_mock_pipeline_isolated(path)
+    println("Pipeline: Starting isolated pipeline...")
+    
+    # 1. Open ISOLATED instance
+    pipeline_msi_data = OpenMSIData(path)
+    println("Pipeline: Opened isolated MSIData instance.")
+    
+    try
+        # 2. Simulate reading spectra
+        indices = 1:min(100, length(pipeline_msi_data.spectra_metadata)) # Read first 100 spectra
+        
+        # Artificial delay to allow "user" interaction
+        sleep(0.5) 
+        
+        println("Pipeline: Reading spectra...")
+        Threads.@threads for i in indices
+            # Use local instance
+            mz, int = GetSpectrum(pipeline_msi_data, i)
+            # Simulate processing work
+            sum(int) 
+        end
+        println("Pipeline: Finished reading spectra successfully.")
+        return true
+    catch e
+        println("Pipeline: CRASHED with error: $e")
+        return false
+    finally
+        close(pipeline_msi_data)
+        println("Pipeline: Closed isolated MSIData instance.")
+    end
+end
+
+# Mock pipeline that uses GLOBAL instance (The BUGGY version)
+function run_mock_pipeline_buggy()
+    println("Buggy Pipeline: Starting...")
+    # Uses global_msi_data directly
+    
+    try
+        global global_msi_data
+        if global_msi_data === nothing
+            println("Buggy Pipeline: No data loaded!")
+            return false
+        end
+        
+        local_ref = global_msi_data # Still points to same object
+        
+        indices = 1:min(100, length(local_ref.spectra_metadata))
+        
+        sleep(0.5)
+        
+        println("Buggy Pipeline: Reading spectra from shared object...")
+        Threads.@threads for i in indices
+            # This will fail if ui_close_file() happens concurrently
+            mz, int = GetSpectrum(local_ref, i)
+            sum(int)
+        end
+        println("Buggy Pipeline: Success (Unexpected if concurrency worked)")
+        return true
+    catch e
+        println("Buggy Pipeline: CRASHED as expected: $e")
+        return false
+    end
+end
+
+
+# --- execute checks ---
+
+@testset "Concurrency Crash Fix Verification" begin
+    
+    # 1. Setup: Load file initially
+    ui_load_file(TEST_FILE)
+    
+    # 2. Test the FIX: Isolated Pipeline
+    println("\n--- Testing Fixed (Isolated) Pipeline ---")
+    
+    t_pipeline = Threads.@spawn run_mock_pipeline_isolated(TEST_FILE)
+    
+    # Simulate user closing/reloading file while pipeline runs
+    sleep(0.2) 
+    ui_close_file()
+    
+    # Wait for pipeline
+    success = fetch(t_pipeline)
+    @test success == true
+    println("Fixed pipeline result: ", success ? "PASSED" : "FAILED")
+
+
+    # 3. Test the BUG: Global Pipeline (Optional, to prove it crashes without fix)
+    # Uncomment to verify the bug exists if needed, but we assume it does based on user report.
+    # println("\n--- Testing Buggy (Shared) Pipeline ---")
+    # ui_load_file(TEST_FILE)
+    # t_buggy = Threads.@spawn run_mock_pipeline_buggy()
+    # sleep(0.2)
+    # ui_close_file()
+    # buggy_success = fetch(t_buggy)
+    # println("Buggy pipeline result: ", buggy_success ? "PASSED (No crash?)" : "FAILED (Crashed as expected)")
+    
+end