dev/Memory_pooling_adjustments #1

Merged
Julian merged 6 commits from dev/Memory_pooling_adjustments into main 2026-05-31 01:03:57 +02:00
2 changed files with 75 additions and 91 deletions
Showing only changes of commit e0ef601a9b - Show all commits

View File

@ -903,6 +903,18 @@ function warmup_init()
end end
end end
"""
clean_registry_path(registry_path::String)
Sanitizes the file path to prevent OS/file system issues, particularly on Windows,
by standardizing separators and removing stripping whitespace.
"""
function clean_registry_path(registry_path::String)
# Standardize to forward slashes internally for uniformity, then normpath formats for OS
clean_path = normpath(strip(replace(registry_path, "\\" => "/")))
return abspath(clean_path)
end
""" """
load_registry(registry_path) load_registry(registry_path)
@ -914,9 +926,9 @@ Loads the dataset registry from a JSON file, with retries for robustness on Wind
# Returns # Returns
- A dictionary containing the registry data. Returns an empty dictionary if the file doesn't exist or fails to parse. - A dictionary containing the registry data. Returns an empty dictionary if the file doesn't exist or fails to parse.
""" """
function load_registry(registry_path) function load_registry(registry_path::String)
return lock(REGISTRY_LOCK) do return lock(REGISTRY_LOCK) do
clean_path = abspath(registry_path) clean_path = clean_registry_path(registry_path)
if isfile(clean_path) if isfile(clean_path)
for attempt in 1:5 for attempt in 1:5
try try
@ -937,93 +949,82 @@ function load_registry(registry_path)
end end
""" """
extract_metadata(msi_data::MSIData, source_path::String) extract_metadata(loaded_data::MSIData, full_route::String)
Extracts key metadata from an `MSIData` object for display. Extracts key metadata from an MSIData object and file path for UI display.
Returns a dictionary with a "summary" key containing parameter-value pairs.
# Arguments
- `msi_data`: The `MSIData` object.
- `source_path`: The path to the source data file.
# Returns
- A dictionary containing summary statistics and metadata.
""" """
function extract_metadata(msi_data::MSIData, source_path::String) function extract_metadata(loaded_data::MSIData, full_route::String)
df = msi_data.spectrum_stats_df stats = []
if df === nothing
# This can happen if precompute_analytics hasn't been run # Basic File Info
# We can still return basic info push!(stats, Dict("parameter" => "Filename", "value" => basename(full_route)))
return Dict( push!(stats, Dict("parameter" => "Source Path", "value" => full_route))
"summary" => [
Dict("parameter" => "File Name", "value" => basename(source_path)), # Image Dimensions
Dict("parameter" => "Number of Spectra", "value" => length(msi_data.spectra_metadata)), w, h = loaded_data.image_dims
Dict("parameter" => "Image Dimensions", "value" => "$(msi_data.image_dims[1]) x $(msi_data.image_dims[2])"), if w > 0 && h > 0
], push!(stats, Dict("parameter" => "Image Dimensions", "value" => "$w x $h"))
"global_min_mz" => nothing, else
"global_max_mz" => nothing push!(stats, Dict("parameter" => "Image Dimensions", "value" => "N/A (Non-imaging)"))
)
end end
summary_stats = [ # Spectrum Counts
Dict("parameter" => "File Name", "value" => basename(source_path)), num_spectra = length(loaded_data.spectra_metadata)
Dict("parameter" => "Number of Spectra", "value" => length(msi_data.spectra_metadata)), push!(stats, Dict("parameter" => "Total Spectra", "value" => string(num_spectra)))
Dict("parameter" => "Image Dimensions", "value" => "$(msi_data.image_dims[1]) x $(msi_data.image_dims[2])"),
Dict("parameter" => "Global Min m/z", "value" => @sprintf("%.4f", Threads.atomic_add!(msi_data.global_min_mz, 0.0))),
Dict("parameter" => "Global Max m/z", "value" => @sprintf("%.4f", Threads.atomic_add!(msi_data.global_max_mz, 0.0))),
Dict("parameter" => "Mean TIC", "value" => @sprintf("%.2e", mean(df.TIC))),
Dict("parameter" => "Mean BPI", "value" => @sprintf("%.2e", mean(df.BPI))),
Dict("parameter" => "Mean # Points", "value" => @sprintf("%.1f", mean(df.NumPoints))),
]
if hasproperty(df, :Mode) # m/z Range
centroid_count = count(==(MSI_src.CENTROID), df.Mode) min_mz, max_mz = get_global_mz_range(loaded_data)
profile_count = count(==(MSI_src.PROFILE), df.Mode) if isfinite(min_mz) && isfinite(max_mz)
unknown_count = count(==(MSI_src.UNKNOWN), df.Mode) push!(stats, Dict("parameter" => "Global m/z Range", "value" => "$(round(min_mz, digits=4)) - $(round(max_mz, digits=4))"))
else
push!(stats, Dict("parameter" => "Global m/z Range", "value" => "Unknown"))
end
push!(summary_stats, Dict("parameter" => "Centroid Spectra", "value" => string(centroid_count))) # Instrument Metadata
push!(summary_stats, Dict("parameter" => "Profile Spectra", "value" => string(profile_count))) if loaded_data.instrument_metadata !== nothing
if unknown_count > 0 inst = loaded_data.instrument_metadata
push!(summary_stats, Dict("parameter" => "Unknown Mode Spectra", "value" => string(unknown_count))) push!(stats, Dict("parameter" => "Instrument Model", "value" => inst.instrument_model))
push!(stats, Dict("parameter" => "Polarity", "value" => titlecase(string(inst.polarity))))
push!(stats, Dict("parameter" => "Acquisition Mode", "value" => titlecase(string(inst.acquisition_mode))))
if inst.resolution !== nothing
push!(stats, Dict("parameter" => "Instrument Resolution", "value" => string(inst.resolution)))
end end
end end
return Dict( return Dict("summary" => stats)
"summary" => summary_stats,
"global_min_mz" => msi_data.global_min_mz,
"global_max_mz" => msi_data.global_max_mz
)
end end
""" """
save_registry(registry_path, registry_data) save_registry(registry_path::String, registry_data::Dict)
Saves the dataset registry to a JSON file, ensuring thread-safe access and robustness on Windows. Saves the dataset registry to a JSON file atomically to prevent data loss
and sharing violations on Windows. Writes to a `.tmp` file first.
# Arguments
- `registry_path`: Path to the `registry.json` file.
- `registry_data`: The dictionary containing the registry data to save.
""" """
function save_registry(registry_path, registry_data) function save_registry(registry_path::String, registry_data)
lock(REGISTRY_LOCK) do lock(REGISTRY_LOCK) do
# Ensure the path is clean and valid for the OS clean_path = clean_registry_path(registry_path)
clean_path = abspath(registry_path)
dir = dirname(clean_path) dir = dirname(clean_path)
if !isdir(dir) if !isdir(dir)
mkpath(dir) mkpath(dir)
end end
# On Windows, opening files for writing can occasionally fail if another process temp_path = clean_path * ".tmp"
# (like an anti-virus or file indexer) holds a transient lock.
# Retry with a small delay to improve robustness. for attempt in 1:6
for attempt in 1:5
try try
open(clean_path, "w") do f open(temp_path, "w") do f
JSON.print(f, registry_data, 4) JSON.print(f, registry_data, 4)
end end
return true # Success
# Replace final file with temp atomically. Force=true maps to ReplaceFile/MoveFileEx
mv(temp_path, clean_path, force=true)
return true
catch e catch e
if attempt == 5 if attempt == 6
@error "Final attempt failed to write to registry.json: $(sprint(showerror, e))" @error "Final attempt failed to write to registry.json: $(sprint(showerror, e))"
isfile(temp_path) && rm(temp_path, force=true)
return false return false
else else
@warn "Attempt $attempt to write to registry.json failed (Error: $(sprint(showerror, e))), retrying in 0.2s..." @warn "Attempt $attempt to write to registry.json failed (Error: $(sprint(showerror, e))), retrying in 0.2s..."
@ -1047,29 +1048,21 @@ Adds or updates an entry in the dataset registry JSON file.
- `metadata`: Optional dictionary of metadata to store. - `metadata`: Optional dictionary of metadata to store.
- `is_imzML`: Boolean indicating if the source is an imzML file. - `is_imzML`: Boolean indicating if the source is an imzML file.
""" """
function update_registry(registry_path, dataset_name, source_path, metadata=nothing, is_imzML=false) function update_registry(registry_path::String, dataset_name::String, source_path::String, metadata=nothing, is_imzML=false)
lock(REGISTRY_LOCK) do lock(REGISTRY_LOCK) do
registry = load_registry(registry_path) registry = load_registry(registry_path)
# Get existing entry if it exists, otherwise create new one
existing_entry = get(registry, dataset_name, Dict{String,Any}()) existing_entry = get(registry, dataset_name, Dict{String,Any}())
# Start with existing data and update only the basic fields
entry = copy(existing_entry) entry = copy(existing_entry)
entry["source_path"] = source_path entry["source_path"] = source_path
entry["processed_date"] = string(now()) entry["processed_date"] = string(now())
entry["is_imzML"] = is_imzML entry["is_imzML"] = is_imzML
# Only update metadata if provided
if metadata !== nothing if metadata !== nothing
entry["metadata"] = metadata entry["metadata"] = metadata
end end
# Note: has_mask and mask_path are preserved from existing_entry if they exist
registry[dataset_name] = entry registry[dataset_name] = entry
# Use the robust save function
save_registry(registry_path, registry) save_registry(registry_path, registry)
end end
end end
@ -1194,24 +1187,14 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov
end end
end end
function update_registry_mask_fields(registry_path, dataset_name, has_mask, mask_path) function update_registry_mask_fields(registry_path::String, dataset_name::String, has_mask::Bool, mask_path::String)
lock(REGISTRY_LOCK) do lock(REGISTRY_LOCK) do
registry = if isfile(registry_path) registry = load_registry(registry_path)
try
JSON.parsefile(registry_path, dicttype=Dict{String,Any})
catch e
@error "Failed to parse registry.json while updating mask fields: $e"
Dict{String,Any}()
end
else
Dict{String,Any}()
end
if haskey(registry, dataset_name) if haskey(registry, dataset_name)
registry[dataset_name]["has_mask"] = has_mask registry[dataset_name]["has_mask"] = has_mask
registry[dataset_name]["mask_path"] = mask_path registry[dataset_name]["mask_path"] = mask_path
else else
# Create new entry if it doesn't exist
registry[dataset_name] = Dict{String,Any}( registry[dataset_name] = Dict{String,Any}(
"has_mask" => has_mask, "has_mask" => has_mask,
"mask_path" => mask_path, "mask_path" => mask_path,
@ -1221,7 +1204,6 @@ function update_registry_mask_fields(registry_path, dataset_name, has_mask, mask
) )
end end
# Use the robust save function
save_registry(registry_path, registry) save_registry(registry_path, registry)
end end
end end

View File

@ -12,7 +12,9 @@ using MSI_src
using .MSI_src: MSIData, process_image_pipeline, REGISTRY_LOCK using .MSI_src: MSIData, process_image_pipeline, REGISTRY_LOCK
# Plot Handling # Plot Handling
if !@isdefined(increment_image)
include("./julia_imzML_visual.jl") include("./julia_imzML_visual.jl")
end
# Image Processing Pipeline # Image Processing Pipeline
using ImageBinarization using ImageBinarization