Implemented more multithreaded precautions in precalculations and preprocessing pipeline, added concurrency tests and validations on several datacentric functions, refractored preprocessing pipeline, added visual upgrades to the UI and a clear session and bug report buttons
This commit is contained in:
Pixelguy14
parent
4414671a30
commit
df94d50954
2757
Manifest.toml
2757
Manifest.toml
File diff suppressed because it is too large
Load Diff
458
app.jl
458
app.jl
@ -8,6 +8,7 @@ using Libz
|
||||
using PlotlyBase
|
||||
using CairoMakie
|
||||
using Colors
|
||||
using Dates
|
||||
using MSI_src # Import the new MSIData library
|
||||
using Statistics
|
||||
using NaturalSort
|
||||
@ -113,7 +114,9 @@ function get_processed_mean_spectrum(spectra::Vector{MutableSpectrum}; num_bins=
|
||||
|
||||
# 3. Bin intensities
|
||||
for s in spectra
|
||||
for i in eachindex(s.mz)
|
||||
# Use minimum length to avoid bounds errors if arrays are mismatched
|
||||
n_points = min(length(s.mz), length(s.intensity))
|
||||
for i in 1:n_points
|
||||
bin_index = trunc(Int, (s.mz[i] - min_mz) * inv_bin_step + 1.0)
|
||||
final_index = clamp(bin_index, 1, num_bins)
|
||||
intensity_sum[final_index] += s.intensity[i]
|
||||
@ -147,7 +150,9 @@ function get_processed_sum_spectrum(spectra::Vector{MutableSpectrum}; num_bins=2
|
||||
inv_bin_step = 1.0 / bin_step
|
||||
|
||||
for s in spectra
|
||||
for i in eachindex(s.mz)
|
||||
# Use minimum length to avoid bounds errors if arrays are mismatched
|
||||
n_points = min(length(s.mz), length(s.intensity))
|
||||
for i in 1:n_points
|
||||
bin_index = trunc(Int, (s.mz[i] - min_mz) * inv_bin_step + 1.0)
|
||||
final_index = clamp(bin_index, 1, num_bins)
|
||||
intensity_sum[final_index] += s.intensity[i]
|
||||
@ -157,11 +162,44 @@ function get_processed_sum_spectrum(spectra::Vector{MutableSpectrum}; num_bins=2
|
||||
return collect(mz_bins), intensity_sum
|
||||
end
|
||||
|
||||
INITIAL_MODEL_STATE = Dict{Symbol,Any}()
|
||||
|
||||
# Function to capture initial state (also outside @app block)
|
||||
function capture_initial_state!(model)
|
||||
empty!(INITIAL_MODEL_STATE)
|
||||
for name in fieldnames(typeof(model))
|
||||
if !startswith(String(name), "_")
|
||||
INITIAL_MODEL_STATE[name] = deepcopy(getfield(model, name))
|
||||
end
|
||||
end
|
||||
println("Captured $(length(INITIAL_MODEL_STATE)) reactive variables")
|
||||
end
|
||||
|
||||
@genietools
|
||||
|
||||
# == Reactive code ==
|
||||
#=
|
||||
macro ui_log(message, level="INFO", log_entries)
|
||||
quote
|
||||
local timestamp = Dates.format(now(), "HH:MM:SS")
|
||||
local new_entry = Dict("time" => timestamp, "message" => string($(esc(message))), "level" => $(esc(level)))
|
||||
println("log entries value: $log_entries")
|
||||
pushfirst!(log_entries, new_entry)
|
||||
if length(log_entries) > 100
|
||||
popfirst!(log_entries)
|
||||
end
|
||||
push!(__model__)
|
||||
end
|
||||
end
|
||||
=#
|
||||
|
||||
# Reactive code to make the UI interactive
|
||||
@app begin
|
||||
# == Notification & Logs ==
|
||||
# @in log_entries = Dict{String,Any}[]
|
||||
# @in show_log_sidebar = false
|
||||
@in showBugModal = false
|
||||
|
||||
# == Loading Screen Variables ==
|
||||
@in is_initializing = true
|
||||
@in initialization_message = "Initializing..."
|
||||
@ -348,6 +386,7 @@ end
|
||||
@in export_params_btn = false # Export parameters to file
|
||||
@in import_params_btn = false # Import parameters from file
|
||||
@in save_feature_matrix_btn = false # Save feature matrix results
|
||||
@in reset_session_btn = false # Deep session reset
|
||||
|
||||
# Preprocessing results
|
||||
@in selected_spectrum_id_for_plot = 1
|
||||
@ -616,12 +655,64 @@ end
|
||||
# Reactive handlers watch a variable and execute a block of code when its value changes
|
||||
# The onbutton handler will set the variable to false after the block is executed
|
||||
|
||||
# This handler correctly uses pick_file and loads the selected file
|
||||
# as the active dataset for the UI.
|
||||
@onbutton reset_session_btn begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
|
||||
try
|
||||
# 1. Clear large data objects explicitly
|
||||
msi_data = nothing
|
||||
feature_matrix_result = nothing
|
||||
bin_info_result = nothing
|
||||
|
||||
# 2. Reset ALL reactive variables using captured initial state
|
||||
if !isempty(INITIAL_MODEL_STATE)
|
||||
for (name, value) in INITIAL_MODEL_STATE
|
||||
setfield!(__model__, name, deepcopy(value))
|
||||
end
|
||||
msg = "Session reset: restored $(length(INITIAL_MODEL_STATE)) variables to initial state."
|
||||
else
|
||||
msg = "Warning: No initial state captured. Using partial reset."
|
||||
end
|
||||
|
||||
# 3. Reset file lists (these will be repopulated by normal operation)
|
||||
msi_bmp = sort(filter(filename -> startswith(filename, "MSI_") && endswith(filename, ".bmp"), readdir("public")), lt=natural)
|
||||
col_msi_png = sort(filter(filename -> startswith(filename, "colorbar_MSI_") && endswith(filename, ".png"), readdir("public")), lt=natural)
|
||||
triq_bmp = sort(filter(filename -> startswith(filename, "TrIQ_") && endswith(filename, ".bmp"), readdir("public")), lt=natural)
|
||||
col_triq_png = sort(filter(filename -> startswith(filename, "colorbar_TrIQ_") && endswith(filename, ".png"), readdir("public")), lt=natural)
|
||||
|
||||
# 4. Clear any cached images/plots
|
||||
imgInt = "/.bmp"
|
||||
imgIntT = "/.bmp"
|
||||
colorbar = "/.png"
|
||||
colorbarT = "/.png"
|
||||
|
||||
# 5. Reset plot data to default traces
|
||||
traceImg = PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}())
|
||||
plotdataImg = [traceImg]
|
||||
plotdataImgT = [traceImg]
|
||||
plotdata = [PlotlyBase.scatter(x=Vector{Float64}(), y=Vector{Float64}(), mode="lines")]
|
||||
|
||||
# 6. Aggressive garbage collection
|
||||
GC.gc(true)
|
||||
if Sys.islinux()
|
||||
ccall(:malloc_trim, Int32, (Int32,), 0)
|
||||
end
|
||||
println("Session reset successfully.")
|
||||
catch e
|
||||
println("Error during session reset: $e")
|
||||
msg = "Reset error: $e"
|
||||
finally
|
||||
is_processing = false
|
||||
end
|
||||
end
|
||||
|
||||
@onbutton btnSearch begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
picked_route = pick_file(; filterlist="imzML,imzml,mzML,mzml")
|
||||
if isempty(picked_route)
|
||||
|
||||
if isnothing(picked_route) || isempty(picked_route)
|
||||
is_processing = false
|
||||
return
|
||||
end
|
||||
@ -938,6 +1029,7 @@ end
|
||||
|
||||
@onbutton export_params_btn begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
params_to_export = Dict(
|
||||
"pipeline_step_order" => pipeline_step_order,
|
||||
"enable_standards" => enable_standards, # Export global flag
|
||||
@ -1011,6 +1103,7 @@ end
|
||||
|
||||
@onbutton import_params_btn begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
picked_file = pick_file(filterlist="json")
|
||||
if isempty(picked_file)
|
||||
is_processing = false
|
||||
@ -1021,39 +1114,57 @@ end
|
||||
json_string = read(picked_file, String)
|
||||
params = JSON.parse(json_string)
|
||||
|
||||
# This is a list of all known reactive variables that can be imported.
|
||||
# This prevents arbitrary variable assignment.
|
||||
known_params = [
|
||||
"stabilization_method", "smoothing_method", "smoothing_window", "smoothing_order",
|
||||
"baseline_method", "baseline_iterations", "baseline_window", "normalization_method",
|
||||
"alignment_method", "alignment_span", "alignment_tolerance", "alignment_tolerance_unit",
|
||||
"alignment_max_shift_ppm", "alignment_min_matched_peaks", "peak_picking_method",
|
||||
"peak_picking_snr_threshold", "peak_picking_half_window", "peak_picking_min_peak_prominence",
|
||||
"peak_picking_merge_peaks_tolerance", "peak_picking_min_peak_width_ppm",
|
||||
"peak_picking_max_peak_width_ppm", "peak_picking_min_peak_shape_r2", "binning_method",
|
||||
"binning_tolerance", "binning_tolerance_unit", "binning_frequency_threshold",
|
||||
"binning_min_peak_per_bin", "binning_max_bin_width_ppm", "binning_intensity_weighted_centers",
|
||||
"binning_num_uniform_bins", "calibration_fit_order", "calibration_ppm_tolerance",
|
||||
"peak_selection_min_snr", "peak_selection_min_fwhm_ppm", "peak_selection_max_fwhm_ppm",
|
||||
"peak_selection_min_shape_r2", "peak_selection_frequency_threshold", "peak_selection_correlation_threshold"
|
||||
]
|
||||
|
||||
for (key, value) in params
|
||||
if key == "reference_peaks_list"
|
||||
reference_peaks_list = value
|
||||
elseif key == "pipeline_step_order"
|
||||
pipeline_step_order = value
|
||||
elseif key == "enable_standards"
|
||||
enable_standards = value
|
||||
elseif key in known_params
|
||||
# Use getfield and setproperty! to update reactive variables by name
|
||||
if hasfield(typeof(@__MODULE__), Symbol(key))
|
||||
getfield(@__MODULE__, Symbol(key))[] = value
|
||||
end
|
||||
else
|
||||
@warn "Unknown parameter '$key' found in JSON file. Skipping."
|
||||
end
|
||||
# Import special variables first
|
||||
if haskey(params, "reference_peaks_list")
|
||||
reference_peaks_list = params["reference_peaks_list"]
|
||||
end
|
||||
if haskey(params, "pipeline_step_order")
|
||||
pipeline_step_order = params["pipeline_step_order"]
|
||||
end
|
||||
if haskey(params, "enable_standards")
|
||||
enable_standards = params["enable_standards"]
|
||||
end
|
||||
|
||||
# Import regular parameters with explicit assignments
|
||||
haskey(params, "stabilization_method") && (stabilization_method = params["stabilization_method"])
|
||||
haskey(params, "smoothing_method") && (smoothing_method = params["smoothing_method"])
|
||||
haskey(params, "smoothing_window") && (smoothing_window = params["smoothing_window"])
|
||||
haskey(params, "smoothing_order") && (smoothing_order = params["smoothing_order"])
|
||||
haskey(params, "baseline_method") && (baseline_method = params["baseline_method"])
|
||||
haskey(params, "baseline_iterations") && (baseline_iterations = params["baseline_iterations"])
|
||||
haskey(params, "baseline_window") && (baseline_window = params["baseline_window"])
|
||||
haskey(params, "normalization_method") && (normalization_method = params["normalization_method"])
|
||||
haskey(params, "alignment_method") && (alignment_method = params["alignment_method"])
|
||||
haskey(params, "alignment_span") && (alignment_span = params["alignment_span"])
|
||||
haskey(params, "alignment_tolerance") && (alignment_tolerance = params["alignment_tolerance"])
|
||||
haskey(params, "alignment_tolerance_unit") && (alignment_tolerance_unit = params["alignment_tolerance_unit"])
|
||||
haskey(params, "alignment_max_shift_ppm") && (alignment_max_shift_ppm = params["alignment_max_shift_ppm"])
|
||||
haskey(params, "alignment_min_matched_peaks") && (alignment_min_matched_peaks = params["alignment_min_matched_peaks"])
|
||||
haskey(params, "peak_picking_method") && (peak_picking_method = params["peak_picking_method"])
|
||||
haskey(params, "peak_picking_snr_threshold") && (peak_picking_snr_threshold = params["peak_picking_snr_threshold"])
|
||||
haskey(params, "peak_picking_half_window") && (peak_picking_half_window = params["peak_picking_half_window"])
|
||||
haskey(params, "peak_picking_min_peak_prominence") && (peak_picking_min_peak_prominence = params["peak_picking_min_peak_prominence"])
|
||||
haskey(params, "peak_picking_merge_peaks_tolerance") && (peak_picking_merge_peaks_tolerance = params["peak_picking_merge_peaks_tolerance"])
|
||||
haskey(params, "peak_picking_min_peak_width_ppm") && (peak_picking_min_peak_width_ppm = params["peak_picking_min_peak_width_ppm"])
|
||||
haskey(params, "peak_picking_max_peak_width_ppm") && (peak_picking_max_peak_width_ppm = params["peak_picking_max_peak_width_ppm"])
|
||||
haskey(params, "peak_picking_min_peak_shape_r2") && (peak_picking_min_peak_shape_r2 = params["peak_picking_min_peak_shape_r2"])
|
||||
haskey(params, "binning_method") && (binning_method = params["binning_method"])
|
||||
haskey(params, "binning_tolerance") && (binning_tolerance = params["binning_tolerance"])
|
||||
haskey(params, "binning_tolerance_unit") && (binning_tolerance_unit = params["binning_tolerance_unit"])
|
||||
haskey(params, "binning_frequency_threshold") && (binning_frequency_threshold = params["binning_frequency_threshold"])
|
||||
haskey(params, "binning_min_peak_per_bin") && (binning_min_peak_per_bin = params["binning_min_peak_per_bin"])
|
||||
haskey(params, "binning_max_bin_width_ppm") && (binning_max_bin_width_ppm = params["binning_max_bin_width_ppm"])
|
||||
haskey(params, "binning_intensity_weighted_centers") && (binning_intensity_weighted_centers = params["binning_intensity_weighted_centers"])
|
||||
haskey(params, "binning_num_uniform_bins") && (binning_num_uniform_bins = params["binning_num_uniform_bins"])
|
||||
haskey(params, "calibration_fit_order") && (calibration_fit_order = params["calibration_fit_order"])
|
||||
haskey(params, "calibration_ppm_tolerance") && (calibration_ppm_tolerance = params["calibration_ppm_tolerance"])
|
||||
haskey(params, "peak_selection_min_snr") && (peak_selection_min_snr = params["peak_selection_min_snr"])
|
||||
haskey(params, "peak_selection_min_fwhm_ppm") && (peak_selection_min_fwhm_ppm = params["peak_selection_min_fwhm_ppm"])
|
||||
haskey(params, "peak_selection_max_fwhm_ppm") && (peak_selection_max_fwhm_ppm = params["peak_selection_max_fwhm_ppm"])
|
||||
haskey(params, "peak_selection_min_shape_r2") && (peak_selection_min_shape_r2 = params["peak_selection_min_shape_r2"])
|
||||
haskey(params, "peak_selection_frequency_threshold") && (peak_selection_frequency_threshold = params["peak_selection_frequency_threshold"])
|
||||
haskey(params, "peak_selection_correlation_threshold") && (peak_selection_correlation_threshold = params["peak_selection_correlation_threshold"])
|
||||
|
||||
msg = "Parameters imported successfully from $(basename(picked_file))."
|
||||
catch e
|
||||
msg = "Failed to import parameters: $e"
|
||||
@ -1110,13 +1221,14 @@ end
|
||||
|
||||
@onbutton run_full_pipeline begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
overall_progress = 0.0
|
||||
local pipeline_msi_data = nothing
|
||||
local current_spectra = Vector{MutableSpectrum}()
|
||||
current_pipeline_step = "Initializing..."
|
||||
println("DEBUG: run_full_pipeline started.")
|
||||
# println("DEBUG: Current pipeline_step_order configuration: $pipeline_step_order")
|
||||
|
||||
try
|
||||
# --- 1. Initial Checks and Data Loading ---
|
||||
println("DEBUG: Performing initial checks and data loading...")
|
||||
if isempty(selected_folder_main)
|
||||
msg = "No dataset loaded. Please load a file using 'Select an imzMl / mzML file'."
|
||||
warning_msg = true
|
||||
@ -1136,26 +1248,31 @@ end
|
||||
target_path = entry["source_path"]
|
||||
|
||||
# Ensure msi_data is for the currently selected file and load if needed
|
||||
if msi_data === nothing || full_route != target_path
|
||||
println("DEBUG: Active file path changed or data not in memory. Reloading MSI data: $(basename(target_path))")
|
||||
if msi_data !== nothing; close(msi_data); end
|
||||
msg = "Reloading $(basename(target_path)) for analysis..."
|
||||
full_route = target_path
|
||||
msi_data = OpenMSIData(target_path)
|
||||
# NOTE: For the pipeline, we will open a DEDICATED instance to avoid race conditions
|
||||
# with the global msi_data used for plotting/interactive exploration.
|
||||
println("DEBUG: Opening isolated MSIData instance for pipeline stability...")
|
||||
pipeline_msi_data = OpenMSIData(target_path)
|
||||
|
||||
# Determine plot mode from loaded data
|
||||
df = msi_data.spectrum_stats_df
|
||||
# Determine plot mode from metadata for correct visualization late
|
||||
metadata = pipeline_msi_data.instrument_metadata
|
||||
acq_mode = metadata !== nothing ? metadata.acquisition_mode : :unknown
|
||||
|
||||
if acq_mode == :centroid
|
||||
last_plot_mode = "stem"
|
||||
elseif acq_mode == :profile
|
||||
last_plot_mode = "lines"
|
||||
else
|
||||
# Fallback to stats if mode is unknown
|
||||
df = pipeline_msi_data.spectrum_stats_df
|
||||
if df !== nothing && "Mode" in names(df)
|
||||
profile_count = count(==(MSI_src.PROFILE), df.Mode)
|
||||
total_count = length(df.Mode)
|
||||
last_plot_mode = profile_count > total_count / 2 ? "lines" : "stem"
|
||||
println("DEBUG: Auto-detected plot mode for pipeline: $(last_plot_mode)")
|
||||
else
|
||||
last_plot_mode = "lines" # Default
|
||||
end
|
||||
else
|
||||
println("DEBUG: Using already loaded MSI data for $(basename(target_path)).")
|
||||
end
|
||||
println("DEBUG: Auto-detected plot mode from metadata: $(last_plot_mode) (acq_mode: $(acq_mode)) [Initial set]")
|
||||
|
||||
# Mask path retrieval from registry
|
||||
local mask_path_for_pipeline::Union{String, Nothing} = nothing
|
||||
@ -1171,6 +1288,7 @@ end
|
||||
warning_msg = true
|
||||
@warn msg
|
||||
println("DEBUG: $msg")
|
||||
close(pipeline_msi_data) # Important cleanup
|
||||
return
|
||||
end
|
||||
else
|
||||
@ -1178,6 +1296,7 @@ end
|
||||
warning_msg = true
|
||||
@warn msg
|
||||
println("DEBUG: $msg")
|
||||
close(pipeline_msi_data) # Important cleanup
|
||||
return
|
||||
end
|
||||
else
|
||||
@ -1185,22 +1304,24 @@ end
|
||||
end
|
||||
|
||||
# Apply mask if enabled to get indices to process
|
||||
spectrum_indices_to_process = collect(1:length(msi_data.spectra_metadata))
|
||||
# Use pipeline_msi_data for consistency
|
||||
spectrum_indices_to_process = collect(1:length(pipeline_msi_data.spectra_metadata))
|
||||
if mask_path_for_pipeline !== nothing
|
||||
current_pipeline_step = "Applying mask..."
|
||||
println("DEBUG: Applying mask matrix to filter spectra...")
|
||||
mask_matrix = load_and_prepare_mask(mask_path_for_pipeline, msi_data.image_dims)
|
||||
masked_indices_set = get_masked_spectrum_indices(msi_data, mask_matrix)
|
||||
mask_matrix = load_and_prepare_mask(mask_path_for_pipeline, pipeline_msi_data.image_dims)
|
||||
masked_indices_set = get_masked_spectrum_indices(pipeline_msi_data, mask_matrix)
|
||||
spectrum_indices_to_process = collect(masked_indices_set)
|
||||
if isempty(spectrum_indices_to_process)
|
||||
msg = "No spectra remaining after applying mask. Aborting pipeline."
|
||||
warning_msg = true
|
||||
println("DEBUG: $msg")
|
||||
close(pipeline_msi_data) # Important cleanup
|
||||
return
|
||||
end
|
||||
println("DEBUG: $(length(spectrum_indices_to_process)) spectra remaining after mask application.")
|
||||
else
|
||||
println("DEBUG: No mask applied. Processing all $(length(msi_data.spectra_metadata)) spectra.")
|
||||
println("DEBUG: No mask applied. Processing all $(length(pipeline_msi_data.spectra_metadata)) spectra.")
|
||||
end
|
||||
|
||||
# Apply subset processing if enabled
|
||||
@ -1211,18 +1332,92 @@ end
|
||||
println("DEBUG: Subset processing enabled. Processing first $(length(spectrum_indices_to_process)) of $n_total spectra.")
|
||||
end
|
||||
|
||||
# Initialize spectra data structure
|
||||
current_pipeline_step = "Loading spectra..."
|
||||
println("DEBUG: Loading $(length(spectrum_indices_to_process)) spectra into MutableSpectrum objects...")
|
||||
current_spectra = Vector{MutableSpectrum}(undef, length(spectrum_indices_to_process))
|
||||
# --- BOUNDS VALIDATION AND DIAGNOSTIC LOGGING ---
|
||||
# Validate all indices are within bounds before attempting to load
|
||||
max_spectra_idx = length(pipeline_msi_data.spectra_metadata)
|
||||
println("DEBUG: Total spectra in dataset: $max_spectra_idx")
|
||||
println("DEBUG: Number of indices to process: $(length(spectrum_indices_to_process))")
|
||||
|
||||
Threads.@threads for i in 1:length(spectrum_indices_to_process)
|
||||
original_idx = spectrum_indices_to_process[i]
|
||||
mz, intensity = GetSpectrum(msi_data, original_idx) # Fetch mz and intensity for the current spectrum
|
||||
current_spectra[i] = MutableSpectrum(original_idx, copy(Float64.(mz)), copy(Float64.(intensity)), NamedTuple{(:mz, :intensity, :fwhm, :shape_r2, :snr, :prominence), NTuple{6, Float64}}[])
|
||||
if !isempty(spectrum_indices_to_process)
|
||||
min_idx = minimum(spectrum_indices_to_process)
|
||||
max_idx = maximum(spectrum_indices_to_process)
|
||||
println("DEBUG: Spectrum indices range: $min_idx to $max_idx")
|
||||
|
||||
# Check for invalid indices
|
||||
invalid_indices = filter(idx -> idx < 1 || idx > max_spectra_idx, spectrum_indices_to_process)
|
||||
if !isempty(invalid_indices)
|
||||
n_invalid = length(invalid_indices)
|
||||
sample_invalid = first(sort(invalid_indices), min(10, n_invalid))
|
||||
msg = "Invalid spectrum indices detected: $n_invalid indices out of range [1, $max_spectra_idx]. First few invalid indices: $sample_invalid"
|
||||
warning_msg = true
|
||||
@error msg
|
||||
println("DEBUG: $msg")
|
||||
close(pipeline_msi_data) # Important cleanup
|
||||
return
|
||||
end
|
||||
println("DEBUG: All spectrum indices are valid (within [1, $max_spectra_idx]).")
|
||||
else
|
||||
println("DEBUG: Warning - spectrum_indices_to_process is empty!")
|
||||
end
|
||||
|
||||
# CRITICAL: Verify indices are unique to prevent race conditions during loading
|
||||
if length(Set(spectrum_indices_to_process)) != length(spectrum_indices_to_process)
|
||||
@warn "Non-unique indices detected in spectrum_indices_to_process. This may cause issues during parallel loading."
|
||||
end
|
||||
|
||||
# Use pipeline_msi_data for reading
|
||||
# Split loading into chunks to update progress bar
|
||||
n_spectra = length(spectrum_indices_to_process)
|
||||
println("DEBUG: Loading $n_spectra spectra into MutableSpectrum objects...")
|
||||
current_spectra = Vector{MutableSpectrum}(undef, n_spectra)
|
||||
|
||||
chunk_size = max(1, n_spectra ÷ 10) # Update progress every 10%
|
||||
for chunk_start in 1:chunk_size:n_spectra
|
||||
chunk_end = min(chunk_start + chunk_size - 1, n_spectra)
|
||||
|
||||
Threads.@threads for i in chunk_start:chunk_end
|
||||
local original_idx = spectrum_indices_to_process[i]
|
||||
local mz, intensity # Enforce thread-local scope
|
||||
|
||||
try
|
||||
mz, intensity = GetSpectrum(pipeline_msi_data, original_idx)
|
||||
|
||||
# Diagnostic check for length mismatch and defensive truncation
|
||||
l_mz = length(mz)
|
||||
l_int = length(intensity)
|
||||
if l_mz != l_int
|
||||
new_len = min(l_mz, l_int)
|
||||
@warn "CRITICAL: Mismatch during loading at index $original_idx. mz=$l_mz, int=$l_int. TRUNCATING."
|
||||
mz = mz[1:new_len]
|
||||
intensity = intensity[1:new_len]
|
||||
end
|
||||
|
||||
current_spectra[i] = MutableSpectrum(original_idx, copy(Float64.(mz)), copy(Float64.(intensity)), NamedTuple{(:mz, :intensity, :fwhm, :shape_r2, :snr, :prominence), NTuple{6, Float64}}[])
|
||||
catch loop_error
|
||||
rethrow(loop_error)
|
||||
end
|
||||
end
|
||||
|
||||
overall_progress = (chunk_end / n_spectra) * 0.2 # Loading is first 20%
|
||||
push!(__model__)
|
||||
end
|
||||
println("DEBUG: All spectra loaded into temporary structure for processing.")
|
||||
|
||||
# We can now close the local MSI data instance as we have loaded everything into memory
|
||||
# However, if we want to support lazy loading scenarios later, we might keep it open.
|
||||
# For now, let's close it here to free up file handles early,
|
||||
# UNLESS `execute_full_preprocessing` needs it (it doesn't seem to based on signature).
|
||||
close(pipeline_msi_data)
|
||||
pipeline_msi_data = nothing # Prevent accidental use
|
||||
|
||||
# Aggressive memory cleanup to return memory to OS
|
||||
println("DEBUG: Performing aggressive memory cleanup...")
|
||||
GC.gc(true) # Full garbage collection with all generations
|
||||
if Sys.islinux()
|
||||
ccall(:malloc_trim, Int32, (Int32,), 0)
|
||||
end
|
||||
println("DEBUG: Closed local pipeline MSIData instance and freed memory.")
|
||||
|
||||
|
||||
# --- 2. Parameter Assembly with Validation ---
|
||||
current_pipeline_step = "Configuring parameters..."
|
||||
@ -1450,8 +1645,16 @@ end
|
||||
mask_path_for_pipeline
|
||||
) do step
|
||||
current_pipeline_step = "Processing: $step"
|
||||
println("DEBUG: Processing step: $step")
|
||||
|
||||
# Update progress based on step index
|
||||
step_idx = findfirst(==(step), pipeline_stp)
|
||||
if step_idx !== nothing
|
||||
# Preprocessing is 20% to 90% (total 70%)
|
||||
overall_progress = 0.2 + (step_idx / length(pipeline_stp)) * 0.7
|
||||
end
|
||||
push!(__model__)
|
||||
end
|
||||
|
||||
println("DEBUG: Pipeline execution finished.")
|
||||
|
||||
# 4. Update Results Display
|
||||
@ -1638,7 +1841,8 @@ end
|
||||
mkpath(output_dir)
|
||||
save_feature_matrix(feature_matrix_result, bin_info_result, output_dir)
|
||||
msg = "Pipeline completed successfully. Feature matrix saved."
|
||||
println("DEBUG: Feature matrix saved to $output_dir")
|
||||
overall_progress = 1.0
|
||||
push!(__model__)
|
||||
else
|
||||
msg = "Pipeline completed successfully. No feature matrix generated (binning step not enabled)."
|
||||
println("DEBUG: $msg")
|
||||
@ -1650,18 +1854,71 @@ end
|
||||
@error "Pipeline failed" exception=(e, catch_backtrace())
|
||||
println("DEBUG: Pipeline caught an exception: $e")
|
||||
finally
|
||||
# Aggressive memory cleanup
|
||||
println("DEBUG: Starting aggressive memory cleanup...")
|
||||
|
||||
# Explicitly clear large data structures
|
||||
try
|
||||
if current_spectra !== nothing && !isempty(current_spectra)
|
||||
# Deep clear individual objects to break references effectively
|
||||
# Use isassigned to prevent UndefRefError if loading failed halfway
|
||||
for i in eachindex(current_spectra)
|
||||
if isassigned(current_spectra, i)
|
||||
s = current_spectra[i]
|
||||
s.mz = Float64[]
|
||||
s.intensity = Float64[]
|
||||
empty!(s.peaks)
|
||||
end
|
||||
end
|
||||
empty!(current_spectra)
|
||||
end
|
||||
current_spectra = nothing
|
||||
|
||||
if feature_matrix_result !== nothing
|
||||
feature_matrix_result = nothing
|
||||
end
|
||||
|
||||
# Close any open pipeline data handles
|
||||
if pipeline_msi_data !== nothing
|
||||
try
|
||||
close(pipeline_msi_data)
|
||||
catch
|
||||
# Already closed, ignore
|
||||
end
|
||||
pipeline_msi_data = nothing
|
||||
end
|
||||
|
||||
println("DEBUG: Data structures cleared. Triggering garbage collection...")
|
||||
catch cleanup_error
|
||||
@warn "Error during data cleanup: $cleanup_error"
|
||||
end
|
||||
|
||||
is_processing = false
|
||||
overall_progress = 0.0
|
||||
current_pipeline_step = ""
|
||||
println("DEBUG: run_full_pipeline finished (finally block).")
|
||||
GC.gc() # Trigger garbage collection
|
||||
|
||||
# Force garbage collection multiple times for thorough cleanup
|
||||
GC.gc()
|
||||
GC.gc() # Second pass to catch any circular references
|
||||
|
||||
# On Linux/Unix, force Julia to return memory to OS
|
||||
if Sys.islinux()
|
||||
ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure Julia returns the freed memory to OS
|
||||
try
|
||||
ccall(:malloc_trim, Int32, (Int32,), 0)
|
||||
println("DEBUG: malloc_trim called successfully (Linux).")
|
||||
catch e
|
||||
@warn "malloc_trim failed: $e"
|
||||
end
|
||||
end
|
||||
|
||||
println("DEBUG: Memory cleanup complete.")
|
||||
end
|
||||
end
|
||||
|
||||
@onbutton recalculate_suggestions_btn begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
if msi_data === nothing
|
||||
msg = "Please load a file first."
|
||||
warning_msg = true
|
||||
@ -1903,6 +2160,7 @@ end
|
||||
# This new handler correctly adds the file from full_route to the batch list.
|
||||
@onbutton btnAddBatch begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
if isempty(full_route) || full_route == "unknown (manually added)"
|
||||
msg = "No active file selected to add to batch."
|
||||
warning_msg = true
|
||||
@ -1923,6 +2181,7 @@ end
|
||||
|
||||
@onbutton clear_batch_btn begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
selected_files = String[]
|
||||
batch_file_count = 0
|
||||
msg = "Batch cleared"
|
||||
@ -1974,6 +2233,7 @@ end
|
||||
|
||||
@onchange btnSearchMzml, btnSearchSync begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
if btnSearchMzml
|
||||
picked_route = pick_file(; filterlist="mzML,mzml")
|
||||
if !isempty(picked_route)
|
||||
@ -1997,6 +2257,7 @@ end
|
||||
|
||||
@onbutton convert_process begin
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
if isempty(mzml_full_route) || isempty(sync_full_route)
|
||||
msg_conversion = "Please select both an .mzML file and a .txt sync file."
|
||||
warning_msg = true
|
||||
@ -2029,6 +2290,7 @@ end
|
||||
@error "Conversion failed" exception=(e, catch_backtrace())
|
||||
finally
|
||||
is_processing = false
|
||||
overall_progress = 0.0
|
||||
# Re-enable button if files are still selected
|
||||
btnConvertDisable = isempty(mzml_full_route) || isempty(sync_full_route)
|
||||
end
|
||||
@ -2061,6 +2323,7 @@ end
|
||||
return
|
||||
end
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
|
||||
masses = Float64[]
|
||||
try
|
||||
@ -2087,7 +2350,8 @@ end
|
||||
|
||||
for (file_idx, file_path) in enumerate(current_selected_files)
|
||||
progress_message = "Processing file $(file_idx)/$(num_files): $(basename(file_path))"
|
||||
overall_progress = current_step / total_steps
|
||||
overall_progress = (file_idx - 1) / num_files
|
||||
push!(__model__)
|
||||
|
||||
all_params = (
|
||||
tolerance = current_tol,
|
||||
@ -2218,6 +2482,10 @@ end
|
||||
end
|
||||
|
||||
@onbutton createMeanPlot @time begin
|
||||
# Pre-initialize for safe cleanup in finally block
|
||||
local xSpectraMz = Vector{Float64}()
|
||||
local ySpectraMz = Vector{Float64}()
|
||||
|
||||
if isempty(selected_folder_main)
|
||||
msg = "No dataset selected. Please process a file and select a folder first."
|
||||
warning_msg = true
|
||||
@ -2225,6 +2493,7 @@ end
|
||||
end
|
||||
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
|
||||
try
|
||||
sTime = time()
|
||||
@ -2275,12 +2544,25 @@ end
|
||||
msg = "Plot loaded in $(eTime) seconds"
|
||||
log_memory_usage("Mean Plot Generated", msi_data)
|
||||
catch e
|
||||
msg = "Could not generate mean spectrum plot: $e"
|
||||
warning_msg = true
|
||||
@error "Mean spectrum plotting failed" exception=(e, catch_backtrace())
|
||||
finally
|
||||
is_processing = false
|
||||
GC.gc() # Trigger garbage collection
|
||||
msg = "Could not generate mean spectrum plot: $e"
|
||||
warning_msg = true
|
||||
@error "Mean spectrum plotting failed" exception=(e, catch_backtrace())
|
||||
finally
|
||||
is_processing = false
|
||||
try
|
||||
if plotdata_before !== nothing
|
||||
plotdata_before = nothing
|
||||
end
|
||||
if !isempty(xSpectraMz)
|
||||
empty!(xSpectraMz)
|
||||
end
|
||||
if !isempty(ySpectraMz)
|
||||
empty!(ySpectraMz)
|
||||
end
|
||||
catch
|
||||
end
|
||||
|
||||
GC.gc() # Trigger garbage collection
|
||||
if Sys.islinux()
|
||||
ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure Julia returns the freed memory to OS
|
||||
end
|
||||
@ -2288,6 +2570,10 @@ end
|
||||
end
|
||||
|
||||
@onbutton createSumPlot @time begin
|
||||
# Pre-initialize for safe cleanup in finally block
|
||||
local xSpectraMz = Vector{Float64}()
|
||||
local ySpectraMz = Vector{Float64}()
|
||||
|
||||
if isempty(selected_folder_main)
|
||||
msg = "No dataset selected. Please process a file and select a folder first."
|
||||
warning_msg = true
|
||||
@ -2295,6 +2581,7 @@ end
|
||||
end
|
||||
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
msg = "Loading total spectrum plot for $(selected_folder_main)..."
|
||||
|
||||
try
|
||||
@ -2350,6 +2637,18 @@ end
|
||||
@error "Total spectrum plotting failed" exception=(e, catch_backtrace())
|
||||
finally
|
||||
is_processing = false
|
||||
try
|
||||
if plotdata_before !== nothing
|
||||
plotdata_before = nothing
|
||||
end
|
||||
if !isempty(xSpectraMz)
|
||||
empty!(xSpectraMz)
|
||||
end
|
||||
if !isempty(ySpectraMz)
|
||||
empty!(ySpectraMz)
|
||||
end
|
||||
catch
|
||||
end
|
||||
GC.gc() # Trigger garbage collection
|
||||
if Sys.islinux()
|
||||
ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure Julia returns the freed memory to OS
|
||||
@ -2365,6 +2664,7 @@ end
|
||||
end
|
||||
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
msg = "Loading plot for $(selected_folder_main)..."
|
||||
|
||||
try
|
||||
@ -2479,6 +2779,7 @@ end
|
||||
end
|
||||
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
msg = "Loading plot for $(selected_folder_main)..."
|
||||
|
||||
try
|
||||
@ -3022,6 +3323,7 @@ end
|
||||
return
|
||||
end
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
|
||||
try
|
||||
# --- Get Mask Path ---
|
||||
@ -3079,6 +3381,7 @@ end
|
||||
end
|
||||
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
|
||||
try
|
||||
# --- Get Mask Path ---
|
||||
@ -3137,6 +3440,7 @@ end
|
||||
end
|
||||
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
|
||||
try
|
||||
sTime=time()
|
||||
@ -3175,6 +3479,7 @@ end
|
||||
end
|
||||
|
||||
is_processing = true
|
||||
push!(__model__)
|
||||
|
||||
try
|
||||
sTime=time()
|
||||
@ -3401,6 +3706,10 @@ end
|
||||
@mounted watchplots()
|
||||
|
||||
@onchange isready begin
|
||||
# Capture state on first run only
|
||||
if isempty(INITIAL_MODEL_STATE)
|
||||
capture_initial_state!(__model__)
|
||||
end
|
||||
# is_processing = true
|
||||
if isready && !registry_init_done
|
||||
sTime=time()
|
||||
@ -3471,6 +3780,5 @@ end
|
||||
end
|
||||
end
|
||||
# == Pages ==
|
||||
# Register a new route and the page that will be loaded on access
|
||||
@page("/", "app.jl.html")
|
||||
end
|
||||
1040
app.jl.html
1040
app.jl.html
File diff suppressed because it is too large
Load Diff
@ -1,6 +1,5 @@
|
||||
# julia_imzML_visual.jl
|
||||
|
||||
const REGISTRY_LOCK = ReentrantLock()
|
||||
|
||||
"""
|
||||
increment_image(current_image, image_list)
|
||||
|
||||
14
mask.jl
14
mask.jl
@ -9,7 +9,7 @@ using Statistics, NaturalSort, LinearAlgebra, StipplePlotly
|
||||
using Base.Filesystem: mv
|
||||
|
||||
using MSI_src
|
||||
using .MSI_src: MSIData, process_image_pipeline
|
||||
using .MSI_src: MSIData, process_image_pipeline, REGISTRY_LOCK
|
||||
|
||||
# Plot Handling
|
||||
include("./julia_imzML_visual.jl")
|
||||
@ -799,8 +799,10 @@ end
|
||||
end
|
||||
|
||||
# Save the updated registry
|
||||
open(reg_path, "w") do f
|
||||
JSON.print(f, registry, 4)
|
||||
lock(REGISTRY_LOCK) do
|
||||
open(reg_path, "w") do f
|
||||
JSON.print(f, registry, 4)
|
||||
end
|
||||
end
|
||||
|
||||
@info "Registry updated with mask: $(final_mask_name)"
|
||||
@ -874,8 +876,10 @@ end
|
||||
|
||||
if !isempty(new_folders) || !isempty(removed_folders)
|
||||
println("Registry changed, saving...")
|
||||
open(reg_path, "w") do f
|
||||
JSON.print(f, registry, 4)
|
||||
lock(REGISTRY_LOCK) do
|
||||
open(reg_path, "w") do f
|
||||
JSON.print(f, registry, 4)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@ -82,30 +82,35 @@ A dramatically simplified buffer pool that avoids complex locking.
|
||||
mutable struct SimpleBufferPool
|
||||
buffers::Dict{Int, Vector{Vector{UInt8}}}
|
||||
max_pool_size::Int
|
||||
lock::ReentrantLock # Add lock for thread safety
|
||||
end
|
||||
|
||||
SimpleBufferPool() = SimpleBufferPool(Dict{Int, Vector{Vector{UInt8}}}(), 50)
|
||||
SimpleBufferPool() = SimpleBufferPool(Dict{Int, Vector{Vector{UInt8}}}(), 50, ReentrantLock())
|
||||
|
||||
function get_buffer!(pool::SimpleBufferPool, size::Int)::Vector{UInt8}
|
||||
# Check for existing buffers of exact size first
|
||||
if haskey(pool.buffers, size) && !isempty(pool.buffers[size])
|
||||
return pop!(pool.buffers[size])
|
||||
lock(pool.lock) do
|
||||
# Check for existing buffers of exact size first
|
||||
if haskey(pool.buffers, size) && !isempty(pool.buffers[size])
|
||||
return pop!(pool.buffers[size])
|
||||
end
|
||||
end
|
||||
|
||||
# No suitable buffer found, allocate new one
|
||||
# No suitable buffer found, allocate new one (outside lock to reduce contention)
|
||||
return Vector{UInt8}(undef, size)
|
||||
end
|
||||
|
||||
function release_buffer!(pool::SimpleBufferPool, buffer::Vector{UInt8})
|
||||
size = length(buffer)
|
||||
|
||||
if !haskey(pool.buffers, size)
|
||||
pool.buffers[size] = Vector{Vector{UInt8}}()
|
||||
end
|
||||
lock(pool.lock) do
|
||||
if !haskey(pool.buffers, size)
|
||||
pool.buffers[size] = Vector{Vector{UInt8}}()
|
||||
end
|
||||
|
||||
# Limit pool size to prevent memory bloat
|
||||
if length(pool.buffers[size]) < pool.max_pool_size
|
||||
push!(pool.buffers[size], buffer)
|
||||
# Limit pool size to prevent memory bloat
|
||||
if length(pool.buffers[size]) < pool.max_pool_size
|
||||
push!(pool.buffers[size], buffer)
|
||||
end
|
||||
end
|
||||
# If pool is full, let buffer get GC'd
|
||||
end
|
||||
|
||||
@ -35,7 +35,12 @@ Atomically seeks to a position and reads data into an array. This is the thread-
|
||||
way to read from a specific offset in the file.
|
||||
"""
|
||||
function read_at!(tsfh::ThreadSafeFileHandle, a::AbstractArray, pos::Integer)
|
||||
if pos < 0
|
||||
throw(ArgumentError("Invalid seek position: $pos"))
|
||||
end
|
||||
lock(tsfh.lock) do
|
||||
# Also check against file size if possible, though filesize() might be expensive to call repeatedly
|
||||
# Let's rely on seek throwing if it goes way out of bounds, but catch the negative case which is definitely an error.
|
||||
seek(tsfh.handle, pos)
|
||||
read!(tsfh.handle, a)
|
||||
end
|
||||
@ -510,6 +515,19 @@ function read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata)
|
||||
mz = Array{source.mz_format}(undef, meta.mz_asset.encoded_length)
|
||||
intensity = Array{source.intensity_format}(undef, meta.int_asset.encoded_length)
|
||||
|
||||
# Validate offsets before reading
|
||||
file_size = filesize(source.ibd_handle)
|
||||
|
||||
mz_end = meta.mz_asset.offset + sizeof(source.mz_format) * meta.mz_asset.encoded_length
|
||||
if meta.mz_asset.offset < 0 || mz_end > file_size
|
||||
throw(FileFormatError("Invalid m/z data offset/length for spectrum $(meta.id): offset=$(meta.mz_asset.offset), end=$mz_end, file_size=$file_size"))
|
||||
end
|
||||
|
||||
int_end = meta.int_asset.offset + sizeof(source.intensity_format) * meta.int_asset.encoded_length
|
||||
if meta.int_asset.offset < 0 || int_end > file_size
|
||||
throw(FileFormatError("Invalid intensity data offset/length for spectrum $(meta.id): offset=$(meta.int_asset.offset), end=$int_end, file_size=$file_size"))
|
||||
end
|
||||
|
||||
# Use the new atomic read_at! method for thread-safety
|
||||
read_at!(source.ibd_handle, mz, meta.mz_asset.offset)
|
||||
read_at!(source.ibd_handle, intensity, meta.int_asset.offset)
|
||||
|
||||
@ -19,7 +19,11 @@ export OpenMSIData,
|
||||
get_global_mz_range,
|
||||
MSIData,
|
||||
_iterate_spectra_fast,
|
||||
validate_spectrum
|
||||
validate_spectrum,
|
||||
REGISTRY_LOCK
|
||||
|
||||
# Define shared registry lock
|
||||
const REGISTRY_LOCK = ReentrantLock()
|
||||
|
||||
# Export the public preprocessing & precalculations API
|
||||
export run_preprocessing_analysis,
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
using StatsBase # For mean, std, median, quantile, mad
|
||||
using Base.Threads # For atomic counters and locking
|
||||
|
||||
# =============================================================================
|
||||
# 7) Spatial & Advanced Processing (Stubs & New Functions)
|
||||
@ -80,11 +81,12 @@ function analyze_mass_accuracy(
|
||||
println("Analyzing mass accuracy for $(length(spectrum_indices)) spectra...")
|
||||
|
||||
all_ppm_errors = Float64[]
|
||||
total_matched_peaks = 0
|
||||
total_spectra_processed = 0
|
||||
total_matched_peaks = Atomic{Int}(0)
|
||||
total_spectra_processed = Atomic{Int}(0)
|
||||
results_lock = ReentrantLock()
|
||||
|
||||
_iterate_spectra_fast(msi_data, spectrum_indices) do idx, mz, intensity
|
||||
total_spectra_processed += 1
|
||||
atomic_add!(total_spectra_processed, 1)
|
||||
if !validate_spectrum(mz, intensity)
|
||||
@warn "Spectrum $idx is invalid, skipping mass accuracy analysis for it."
|
||||
return
|
||||
@ -106,8 +108,10 @@ function analyze_mass_accuracy(
|
||||
end
|
||||
|
||||
if best_matched_peak_mz !== nothing
|
||||
push!(all_ppm_errors, min_ppm_error)
|
||||
total_matched_peaks += 1
|
||||
lock(results_lock) do
|
||||
push!(all_ppm_errors, min_ppm_error)
|
||||
end
|
||||
atomic_add!(total_matched_peaks, 1)
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -140,8 +144,8 @@ function analyze_mass_accuracy(
|
||||
std_ppm_error = std_err,
|
||||
min_ppm_error = min_err,
|
||||
max_ppm_error = max_err,
|
||||
total_matched_peaks = total_matched_peaks,
|
||||
total_spectra_analyzed = total_spectra_processed,
|
||||
total_matched_peaks = total_matched_peaks[],
|
||||
total_spectra_analyzed = total_spectra_processed[],
|
||||
ppm_error_distribution = all_ppm_errors
|
||||
)
|
||||
end
|
||||
@ -318,21 +322,29 @@ function analyze_instrument_characteristics(msi_data::MSIData; sample_indices::A
|
||||
return results
|
||||
end
|
||||
|
||||
results_lock = ReentrantLock()
|
||||
|
||||
_iterate_spectra_fast(msi_data, sample_indices) do idx, mz, intensity
|
||||
# Record spectrum mode
|
||||
push!(spectrum_modes, msi_data.spectra_metadata[idx].mode)
|
||||
lock(results_lock) do
|
||||
push!(spectrum_modes, msi_data.spectra_metadata[idx].mode)
|
||||
end
|
||||
|
||||
# Calculate m/z step statistics (for profile data)
|
||||
if length(mz) > 1 && msi_data.spectra_metadata[idx].mode == PROFILE
|
||||
steps = diff(mz)
|
||||
if !isempty(steps)
|
||||
push!(mz_step_sizes, mean(steps))
|
||||
lock(results_lock) do
|
||||
push!(mz_step_sizes, mean(steps))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Record intensity range
|
||||
if !isempty(intensity)
|
||||
push!(intensity_ranges, (minimum(intensity), maximum(intensity)))
|
||||
lock(results_lock) do
|
||||
push!(intensity_ranges, (minimum(intensity), maximum(intensity)))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -442,11 +454,15 @@ function analyze_signal_quality(msi_data::MSIData; sample_indices::AbstractVecto
|
||||
return results
|
||||
end
|
||||
|
||||
results_lock = ReentrantLock()
|
||||
|
||||
_iterate_spectra_fast(msi_data, sample_indices) do idx, mz, intensity
|
||||
if !isempty(intensity)
|
||||
# Noise estimation using MAD
|
||||
noise = mad(intensity, normalize=true)
|
||||
push!(noise_levels, noise)
|
||||
lock(results_lock) do
|
||||
push!(noise_levels, noise)
|
||||
end
|
||||
|
||||
# FIX: More robust SNR calculation
|
||||
valid_intensity = intensity[intensity .> 0] # Remove zeros
|
||||
@ -458,12 +474,16 @@ function analyze_signal_quality(msi_data::MSIData; sample_indices::AbstractVecto
|
||||
if noise_robust > 0 && isfinite(signal_estimate)
|
||||
snr_val = signal_estimate / noise_robust
|
||||
# Cap unrealistic SNR values
|
||||
push!(snr_distribution, min(snr_val, 1e6))
|
||||
lock(results_lock) do
|
||||
push!(snr_distribution, min(snr_val, 1e6))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Total ion count
|
||||
push!(tic_values, sum(intensity))
|
||||
lock(results_lock) do
|
||||
push!(tic_values, sum(intensity))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -651,20 +671,23 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
|
||||
peak_counts = Int[]
|
||||
fwhm_values = Float64[]
|
||||
|
||||
spectra_analyzed = 0
|
||||
peaks_analyzed = 0
|
||||
spectra_analyzed = Atomic{Int}(0)
|
||||
peaks_analyzed = Atomic{Int}(0)
|
||||
results_lock = ReentrantLock()
|
||||
|
||||
_iterate_spectra_fast(msi_data, spectrum_indices) do idx, mz, intensity
|
||||
if length(mz) < 10 # Skip spectra with too few points
|
||||
return
|
||||
end
|
||||
|
||||
spectra_analyzed += 1
|
||||
atomic_add!(spectra_analyzed, 1)
|
||||
meta = msi_data.spectra_metadata[idx]
|
||||
|
||||
# Detect peaks with lower SNR threshold to find more peaks
|
||||
peaks = detect_peaks_profile_core(mz, intensity; snr_threshold=2.0)
|
||||
push!(peak_counts, length(peaks))
|
||||
lock(results_lock) do
|
||||
push!(peak_counts, length(peaks))
|
||||
end
|
||||
|
||||
if !isempty(peaks)
|
||||
# Analyze the strongest 3 peaks per spectrum
|
||||
@ -679,15 +702,17 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
|
||||
if !isnan(fwhm_delta_m) && fwhm_delta_m > 0.001 && fwhm_delta_m < 0.5 # Reasonable range in Da
|
||||
fwhm_ppm = 1e6 * fwhm_delta_m / peak.mz
|
||||
if 5.0 < fwhm_ppm < 500.0 # Reasonable ppm range
|
||||
push!(peak_widths_ppm, fwhm_ppm)
|
||||
push!(fwhm_values, fwhm_delta_m)
|
||||
lock(results_lock) do
|
||||
push!(peak_widths_ppm, fwhm_ppm)
|
||||
push!(fwhm_values, fwhm_delta_m)
|
||||
|
||||
r2 = _fit_gaussian_and_r2(mz, intensity, peak_idx, 5)
|
||||
push!(r_squared_values, r2)
|
||||
peaks_analyzed += 1
|
||||
r2 = _fit_gaussian_and_r2(mz, intensity, peak_idx, 5)
|
||||
push!(r_squared_values, r2)
|
||||
end
|
||||
atomic_add!(peaks_analyzed, 1)
|
||||
|
||||
if peaks_analyzed <= 3
|
||||
println("DEBUG: Peak at m/z $(peak.mz), FWHM = $(fwhm_ppm) ppm, R² = $r2")
|
||||
if peaks_analyzed[] <= 3
|
||||
println("DEBUG: Peak at m/z $(peak.mz), FWHM = $(fwhm_ppm) ppm, R² = $(r_squared_values[end])")
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -698,7 +723,7 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
|
||||
end
|
||||
end
|
||||
|
||||
println("DEBUG: Analyzed $peaks_analyzed peaks from $spectra_analyzed spectra")
|
||||
println("DEBUG: Analyzed $(peaks_analyzed[]) peaks from $(spectra_analyzed[]) spectra")
|
||||
|
||||
if !isempty(peak_widths_ppm)
|
||||
results[:mean_fwhm_ppm] = mean(peak_widths_ppm)
|
||||
@ -727,9 +752,13 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
|
||||
|
||||
peak_counts = Int[]
|
||||
|
||||
results_lock = ReentrantLock()
|
||||
|
||||
_iterate_spectra_fast(msi_data, spectrum_indices) do idx, mz, intensity
|
||||
if !isempty(mz)
|
||||
push!(peak_counts, length(mz))
|
||||
lock(results_lock) do
|
||||
push!(peak_counts, length(mz))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@ -33,7 +33,15 @@ function apply_baseline_correction(spectra::Vector{MutableSpectrum}, params::Dic
|
||||
|
||||
Threads.@threads for s in spectra
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
original_length = length(s.intensity)
|
||||
baseline = apply_baseline_correction_core(s.intensity; method=method, iterations=iterations, window=window)
|
||||
|
||||
# CRITICAL: Ensure baseline has the same length as intensity
|
||||
if length(baseline) != original_length
|
||||
@warn "Baseline correction: length mismatch for spectrum $(s.id). baseline=$(length(baseline)), intensity=$original_length. Skipping this spectrum."
|
||||
continue
|
||||
end
|
||||
|
||||
s.intensity = max.(0.0, s.intensity .- baseline)
|
||||
end
|
||||
end
|
||||
@ -53,7 +61,16 @@ function apply_intensity_transformation(spectra::Vector{MutableSpectrum}, params
|
||||
|
||||
Threads.@threads for s in spectra
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
s.intensity = transform_intensity_core(s.intensity; method=method)
|
||||
original_length = length(s.intensity)
|
||||
transformed = transform_intensity_core(s.intensity; method=method)
|
||||
|
||||
# CRITICAL: Ensure transformation preserves array length
|
||||
if length(transformed) != original_length
|
||||
@warn "Intensity transformation: length mismatch for spectrum $(s.id). transformed=$(length(transformed)), original=$original_length. Skipping this spectrum."
|
||||
continue
|
||||
end
|
||||
|
||||
s.intensity = transformed
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -76,8 +93,16 @@ function apply_smoothing(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
|
||||
Threads.@threads for s in spectra
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
smoothed_intensity = max.(0.0, smooth_spectrum_core(s.intensity; method=method, window=window, order=order))
|
||||
s.intensity = smoothed_intensity
|
||||
original_length = length(s.intensity)
|
||||
smoothed_intensity = smooth_spectrum_core(s.intensity; method=method, window=window, order=order)
|
||||
|
||||
# CRITICAL: Ensure smoothing preserves array length
|
||||
if length(smoothed_intensity) != original_length
|
||||
@warn "Smoothing: length mismatch for spectrum $(s.id). smoothed=$(length(smoothed_intensity)), original=$original_length, method=$method, window=$window. Skipping this spectrum."
|
||||
continue
|
||||
end
|
||||
|
||||
s.intensity = max.(0.0, smoothed_intensity)
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -104,6 +129,7 @@ function apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
|
||||
Threads.@threads for s in spectra
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
# old_len = length(s.peaks)
|
||||
if method == :profile
|
||||
s.peaks = detect_peaks_profile_core(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window, min_peak_prominence=min_peak_prominence, merge_peaks_tolerance=merge_peaks_tolerance)
|
||||
elseif method == :wavelet
|
||||
@ -113,6 +139,7 @@ function apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
else
|
||||
s.peaks = detect_peaks_profile_core(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
|
||||
end
|
||||
# @info "Spectrum $(s.id): detected $(length(s.peaks)) peaks"
|
||||
else
|
||||
s.peaks = []
|
||||
end
|
||||
@ -182,12 +209,21 @@ function apply_calibration(spectra::Vector{MutableSpectrum}, params::Dict, refer
|
||||
Threads.@threads for i in 1:length(spectra)
|
||||
s = spectra[i]
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
original_length = length(s.mz)
|
||||
matched_peaks = find_calibration_peaks_core(s.mz, s.intensity, reference_masses; ppm_tolerance=ppm_tolerance)
|
||||
if length(matched_peaks) >= 2
|
||||
measured = sort(collect(values(matched_peaks)))
|
||||
theoretical = sort(collect(keys(matched_peaks)))
|
||||
itp = linear_interpolation(measured, theoretical, extrapolation_bc=Line())
|
||||
s.mz = itp(s.mz) # Modify mz-axis in-place
|
||||
new_mz = itp(s.mz)
|
||||
|
||||
# CRITICAL: Ensure m/z axis preserves array length
|
||||
if length(new_mz) != original_length
|
||||
@warn "Calibration: length mismatch for spectrum $(s.id). new_mz=$(length(new_mz)), original=$original_length. Skipping this spectrum."
|
||||
continue
|
||||
end
|
||||
|
||||
s.mz = new_mz # Modify mz-axis in-place
|
||||
else
|
||||
@warn "Spectrum $(s.id): insufficient reference peaks ($(length(matched_peaks)) found), skipping calibration."
|
||||
end
|
||||
@ -233,7 +269,16 @@ function apply_peak_alignment(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
current_peaks_mz = [p.mz for p in s.peaks]
|
||||
alignment_func = align_peaks_lowess_core(ref_peaks_mz, current_peaks_mz; method=method, tolerance=tolerance, tolerance_unit=tolerance_unit)
|
||||
|
||||
s.mz = alignment_func.(s.mz) # Update m/z axis
|
||||
original_length = length(s.mz)
|
||||
new_mz = alignment_func.(s.mz)
|
||||
|
||||
# CRITICAL: Ensure alignment preserves array length
|
||||
if length(new_mz) != original_length
|
||||
@warn "Peak alignment: m/z length mismatch for spectrum $(s.id). new_mz=$(length(new_mz)), original=$original_length. Skipping this spectrum."
|
||||
continue
|
||||
end
|
||||
|
||||
s.mz = new_mz # Update m/z axis
|
||||
|
||||
# Update peak m/z values
|
||||
for i in 1:length(s.peaks)
|
||||
@ -258,7 +303,16 @@ function apply_normalization(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
|
||||
Threads.@threads for s in spectra
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
s.intensity = apply_normalization_core(s.intensity; method=method)
|
||||
original_length = length(s.intensity)
|
||||
normalized = apply_normalization_core(s.intensity; method=method)
|
||||
|
||||
# CRITICAL: Ensure normalization preserves array length
|
||||
if length(normalized) != original_length
|
||||
@warn "Normalization: length mismatch for spectrum $(s.id). normalized=$(length(normalized)), original=$original_length. Skipping this spectrum."
|
||||
continue
|
||||
end
|
||||
|
||||
s.intensity = normalized
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -6,17 +6,14 @@ ENV["GENIE_ENV"] = "dev"
|
||||
|
||||
manifest_path = joinpath(@__DIR__, "Manifest.toml")
|
||||
|
||||
# Only instantiate in development mode
|
||||
if get(ENV, "GENIE_ENV", "dev") != "prod" || !isfile(manifest_path)
|
||||
@info "Development environment detected. Instantiating packages..."
|
||||
|
||||
if !isfile(manifest_path)
|
||||
@info "Manifest.toml not found. Generating it based on Project.toml..."
|
||||
end
|
||||
|
||||
# Selective instantiation for faster startup
|
||||
if get(ENV, "GENIE_ENV", "dev") != "prod" && !isfile(manifest_path)
|
||||
@info "Development environment detected and Manifest.toml missing. Instantiating packages..."
|
||||
Pkg.resolve()
|
||||
Pkg.instantiate()
|
||||
Pkg.gc()
|
||||
elseif get(ENV, "GENIE_ENV", "dev") != "prod"
|
||||
@info "Manifest.toml found. Skipping Pkg.instantiate() for faster boot. Delete Manifest.toml if you need to re-instantiate."
|
||||
end
|
||||
|
||||
using Genie
|
||||
|
||||
32
test/test_buffer_pool_concurrency.jl
Normal file
32
test/test_buffer_pool_concurrency.jl
Normal file
@ -0,0 +1,32 @@
|
||||
using MSI_src
|
||||
using Test
|
||||
using Base.Threads
|
||||
|
||||
@testset "SimpleBufferPool Concurrency Stress Test" begin
|
||||
pool = MSI_src.SimpleBufferPool()
|
||||
n_threads = Threads.nthreads()
|
||||
n_iterations = 1000
|
||||
buffer_size = 1024
|
||||
|
||||
println("Running buffer pool stress test with $n_threads threads...")
|
||||
|
||||
# Parallel stress test
|
||||
Threads.@threads for i in 1:(n_threads * n_iterations)
|
||||
# get_buffer! and release_buffer! are now thread-safe
|
||||
buf = MSI_src.get_buffer!(pool, buffer_size)
|
||||
|
||||
# Simulate some work
|
||||
fill!(buf, UInt8(i % 256))
|
||||
|
||||
MSI_src.release_buffer!(pool, buf)
|
||||
end
|
||||
|
||||
# After stress test, the dictionary should be coherent
|
||||
sizes = collect(keys(pool.buffers))
|
||||
if !isempty(sizes)
|
||||
@test buffer_size ∈ sizes
|
||||
@test length(pool.buffers[buffer_size]) <= pool.max_pool_size
|
||||
end
|
||||
|
||||
println("Buffer pool stress test PASSED.")
|
||||
end
|
||||
146
test/test_pipeline_concurrency.jl
Normal file
146
test/test_pipeline_concurrency.jl
Normal file
@ -0,0 +1,146 @@
|
||||
|
||||
using Pkg
|
||||
Pkg.activate(joinpath(@__DIR__, ".."))
|
||||
using MSI_src
|
||||
using Test
|
||||
using Base.Threads
|
||||
|
||||
# --- Test Configuration ---
|
||||
const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Stomach/Stomach_DHB_uncompressed.imzML"
|
||||
|
||||
println("Starting concurrency test with file: $TEST_FILE")
|
||||
|
||||
if !isfile(TEST_FILE)
|
||||
error("Test file not found: $TEST_FILE")
|
||||
end
|
||||
|
||||
# --- Simulation of the Concurrency Issue ---
|
||||
|
||||
# Global "UI State" variable simulates the app's msi_data
|
||||
global_msi_data = nothing
|
||||
const GLOBAL_LOCK = ReentrantLock()
|
||||
|
||||
function ui_load_file(path)
|
||||
global global_msi_data
|
||||
lock(GLOBAL_LOCK) do
|
||||
if global_msi_data !== nothing
|
||||
close(global_msi_data)
|
||||
end
|
||||
println("UI: Loading file...")
|
||||
global_msi_data = OpenMSIData(path)
|
||||
println("UI: File loaded.")
|
||||
end
|
||||
end
|
||||
|
||||
function ui_close_file()
|
||||
global global_msi_data
|
||||
lock(GLOBAL_LOCK) do
|
||||
if global_msi_data !== nothing
|
||||
println("UI: Closing file...")
|
||||
close(global_msi_data)
|
||||
global_msi_data = nothing
|
||||
println("UI: File closed.")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Mock pipeline function that mimics `run_full_pipeline` in app.jl
|
||||
# Crucially, it uses its own local `pipeline_msi_data` as per the fix
|
||||
function run_mock_pipeline_isolated(path)
|
||||
println("Pipeline: Starting isolated pipeline...")
|
||||
|
||||
# 1. Open ISOLATED instance
|
||||
pipeline_msi_data = OpenMSIData(path)
|
||||
println("Pipeline: Opened isolated MSIData instance.")
|
||||
|
||||
try
|
||||
# 2. Simulate reading spectra
|
||||
indices = 1:min(100, length(pipeline_msi_data.spectra_metadata)) # Read first 100 spectra
|
||||
|
||||
# Artificial delay to allow "user" interaction
|
||||
sleep(0.5)
|
||||
|
||||
println("Pipeline: Reading spectra...")
|
||||
Threads.@threads for i in indices
|
||||
# Use local instance
|
||||
mz, int = GetSpectrum(pipeline_msi_data, i)
|
||||
# Simulate processing work
|
||||
sum(int)
|
||||
end
|
||||
println("Pipeline: Finished reading spectra successfully.")
|
||||
return true
|
||||
catch e
|
||||
println("Pipeline: CRASHED with error: $e")
|
||||
return false
|
||||
finally
|
||||
close(pipeline_msi_data)
|
||||
println("Pipeline: Closed isolated MSIData instance.")
|
||||
end
|
||||
end
|
||||
|
||||
# Mock pipeline that uses GLOBAL instance (The BUGGY version)
|
||||
function run_mock_pipeline_buggy()
|
||||
println("Buggy Pipeline: Starting...")
|
||||
# Uses global_msi_data directly
|
||||
|
||||
try
|
||||
global global_msi_data
|
||||
if global_msi_data === nothing
|
||||
println("Buggy Pipeline: No data loaded!")
|
||||
return false
|
||||
end
|
||||
|
||||
local_ref = global_msi_data # Still points to same object
|
||||
|
||||
indices = 1:min(100, length(local_ref.spectra_metadata))
|
||||
|
||||
sleep(0.5)
|
||||
|
||||
println("Buggy Pipeline: Reading spectra from shared object...")
|
||||
Threads.@threads for i in indices
|
||||
# This will fail if ui_close_file() happens concurrently
|
||||
mz, int = GetSpectrum(local_ref, i)
|
||||
sum(int)
|
||||
end
|
||||
println("Buggy Pipeline: Success (Unexpected if concurrency worked)")
|
||||
return true
|
||||
catch e
|
||||
println("Buggy Pipeline: CRASHED as expected: $e")
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# --- execute checks ---
|
||||
|
||||
@testset "Concurrency Crash Fix Verification" begin
|
||||
|
||||
# 1. Setup: Load file initially
|
||||
ui_load_file(TEST_FILE)
|
||||
|
||||
# 2. Test the FIX: Isolated Pipeline
|
||||
println("\n--- Testing Fixed (Isolated) Pipeline ---")
|
||||
|
||||
t_pipeline = Threads.@spawn run_mock_pipeline_isolated(TEST_FILE)
|
||||
|
||||
# Simulate user closing/reloading file while pipeline runs
|
||||
sleep(0.2)
|
||||
ui_close_file()
|
||||
|
||||
# Wait for pipeline
|
||||
success = fetch(t_pipeline)
|
||||
@test success == true
|
||||
println("Fixed pipeline result: ", success ? "PASSED" : "FAILED")
|
||||
|
||||
|
||||
# 3. Test the BUG: Global Pipeline (Optional, to prove it crashes without fix)
|
||||
# Uncomment to verify the bug exists if needed, but we assume it does based on user report.
|
||||
# println("\n--- Testing Buggy (Shared) Pipeline ---")
|
||||
# ui_load_file(TEST_FILE)
|
||||
# t_buggy = Threads.@spawn run_mock_pipeline_buggy()
|
||||
# sleep(0.2)
|
||||
# ui_close_file()
|
||||
# buggy_success = fetch(t_buggy)
|
||||
# println("Buggy pipeline result: ", buggy_success ? "PASSED (No crash?)" : "FAILED (Crashed as expected)")
|
||||
|
||||
end
|
||||
Loading…
x
Reference in New Issue
Block a user