Julian/JuliaMSI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implementing foundations for multithreading, standarized mzml converter, profile/centroid detection per pixel, memory-efficient buffer reuse, added preparations to bloom filter alongside buffer pulling, removed memory leaks on linux due to @async macro used on @app module, better error management

Browse Source
This commit is contained in:
Pixelguy14 2025-11-06 16:43:34 -06:00
parent 3439a8a4dd
commit a8c2dfcc91
14 changed files with 1350 additions and 1026 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Manifest.toml
View File

@ -2,7 +2,7 @@
julia_version = "1.11.7" julia_version = "1.11.7"
manifest_format = "2.0" manifest_format = "2.0"
project_hash = "9b36a3561cfc9927071202de68baf8c2b0f02a5c" project_hash = "f5ab935406d0b7b7e657445ecfa45944ea71f247"
[[deps.ATK_jll]] [[deps.ATK_jll]]
deps = ["Artifacts", "Glib_jll", "JLLWrappers", "Libdl"] deps = ["Artifacts", "Glib_jll", "JLLWrappers", "Libdl"]

2
Project.toml
View File

@ -32,8 +32,10 @@ NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4"
NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85" NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85"
PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5" PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
SavitzkyGolay = "c4bf5708-b6a6-4fbe-bcd0-6850ed671584" SavitzkyGolay = "c4bf5708-b6a6-4fbe-bcd0-6850ed671584"
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
StipplePlotly = "ec984513-233d-481d-95b0-a3b58b97af2b" StipplePlotly = "ec984513-233d-481d-95b0-a3b58b97af2b"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

2
README.md
View File

@ -22,7 +22,7 @@ https://codeberg.org/LabABI/JuliaMSI
~/Downloads/JuliaMSI-main/juliamsi ~/Downloads/JuliaMSI-main/juliamsi
2. Without entering the Julia environment, launch the project in your terminal with the following command (which works for all operating systems): 2. Without entering the Julia environment, launch the project in your terminal with the following command (which works for all operating systems):
``` ```
julia --project=. start_MSI_GUI.jl julia --threads auto --project=. start_MSI_GUI.jl
``` ```
3. After the script has finished loading, you can open a [page](http://127.0.0.1:1481/) in your browser with the web app running. 3. After the script has finished loading, you can open a [page](http://127.0.0.1:1481/) in your browser with the web app running.

1296
app.jl
View File

File diff suppressed because it is too large Load Diff

198
julia_imzML_visual.jl
View File

@ -1,5 +1,7 @@
# julia_imzML_visual.jl # julia_imzML_visual.jl
const REGISTRY_LOCK = ReentrantLock()
""" """
increment_image(current_image, image_list) increment_image(current_image, image_list)
@ -541,7 +543,17 @@ function meanSpectrumPlot(data::MSIData, dataset_name::String=""; mask_path::Uni
# Update title to indicate empty spectrum # Update title to indicate empty spectrum
layout.title.text = "Empty " * layout.title.text layout.title.text = "Empty " * layout.title.text
else else
trace = PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz, marker=attr(size=1, color="blue", opacity=0.5), name="Average", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>") df = data.spectrum_stats_df
profile_count = 0
if df !== nothing && hasproperty(df, :Mode)
profile_count = count(==(MSI_src.PROFILE), df.Mode)
end
if profile_count > 0
trace = PlotlyBase.scatter(x=xSpectraMz, y=ySpectraMz, mode="lines", marker=attr(size=1, color="blue", opacity=0.5), name="Average", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
else
trace = PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz, marker=attr(size=1, color="blue", opacity=0.5), name="Average", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
end
end end
plotdata = [trace] plotdata = [trace]
@ -571,6 +583,7 @@ Generates a plot for the spectrum at a specific coordinate (for imaging data) or
function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int, imgHeight::Int, dataset_name::String=""; mask_path::Union{String, Nothing}=nothing) function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int, imgHeight::Int, dataset_name::String=""; mask_path::Union{String, Nothing}=nothing)
local mz::AbstractVector, intensity::AbstractVector local mz::AbstractVector, intensity::AbstractVector
local plot_title::String local plot_title::String
local spectrum_mode = MSI_src.CENTROID # Default to centroid
is_imaging = data.source isa ImzMLSource is_imaging = data.source isa ImzMLSource
@ -578,6 +591,17 @@ function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int,
x = clamp(xCoord, 1, imgWidth) x = clamp(xCoord, 1, imgWidth)
y = clamp(yCoord, 1, imgHeight) y = clamp(yCoord, 1, imgHeight)
# Get spectrum mode
if data.spectrum_stats_df !== nothing && hasproperty(data.spectrum_stats_df, :Mode)
w, h = data.image_dims
if y > 0 && x > 0 && y <= h && x <= w
idx = (y - 1) * w + x
if idx <= length(data.spectrum_stats_df.Mode)
spectrum_mode = data.spectrum_stats_df.Mode[idx]
end
end
end
if mask_path !== nothing if mask_path !== nothing
mask_matrix = load_and_prepare_mask(mask_path, (imgWidth, imgHeight)) mask_matrix = load_and_prepare_mask(mask_path, (imgWidth, imgHeight))
if !mask_matrix[y, x] if !mask_matrix[y, x]
@ -602,6 +626,12 @@ function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int,
# For non-imaging data, treat xCoord as the spectrum index # For non-imaging data, treat xCoord as the spectrum index
index = clamp(xCoord, 1, length(data.spectra_metadata)) index = clamp(xCoord, 1, length(data.spectra_metadata))
if data.spectrum_stats_df !== nothing && hasproperty(data.spectrum_stats_df, :Mode)
if index <= length(data.spectrum_stats_df.Mode)
spectrum_mode = data.spectrum_stats_df.Mode[index]
end
end
process_spectrum(data, index) do recieved_mz, recieved_intensity process_spectrum(data, index) do recieved_mz, recieved_intensity
mz = recieved_mz mz = recieved_mz
intensity = recieved_intensity intensity = recieved_intensity
@ -636,7 +666,11 @@ function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int,
# Downsample for plotting performance # Downsample for plotting performance
mz_down, int_down = MSI_src.downsample_spectrum(mz, intensity) mz_down, int_down = MSI_src.downsample_spectrum(mz, intensity)
trace = PlotlyBase.stem(x=mz_down, y=int_down, marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>") trace = if spectrum_mode == MSI_src.PROFILE
PlotlyBase.scatter(x=mz_down, y=int_down, mode="lines", marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
else
PlotlyBase.stem(x=mz_down, y=int_down, marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
end
plotdata = [trace] plotdata = [trace]
plotlayout = layout plotlayout = layout
@ -700,7 +734,17 @@ function sumSpectrumPlot(data::MSIData, dataset_name::String=""; mask_path::Unio
# Update title to indicate empty spectrum # Update title to indicate empty spectrum
layout.title.text = "Empty " * layout.title.text layout.title.text = "Empty " * layout.title.text
else else
trace = PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz, marker=attr(size=1, color="blue", opacity=0.5), name="Total", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>") df = data.spectrum_stats_df
profile_count = 0
if df !== nothing && hasproperty(df, :Mode)
profile_count = count(==(MSI_src.PROFILE), df.Mode)
end
if profile_count > 0
trace = PlotlyBase.scatter(x=xSpectraMz, y=ySpectraMz, mode="lines", marker=attr(size=1, color="blue", opacity=0.5), name="Total", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
else
trace = PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz, marker=attr(size=1, color="blue", opacity=0.5), name="Total", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
end
end end
plotdata = [trace] plotdata = [trace]
@ -733,7 +777,7 @@ function warmup_init()
try try
save_bitmap(dummy_bmp_path, zeros(UInt8, 10, 10), ViridisPalette) save_bitmap(dummy_bmp_path, zeros(UInt8, 10, 10), ViridisPalette)
loadImgPlot("/dummy.bmp") loadImgPlot("/dummy.bmp")
generate_colorbar_image(zeros(10, 10), 256, dummy_png_path) generate_colorbar_image(zeros(10, 10), 256, dummy_png_path, (0.0, 1.0))
catch e catch e
@warn "Pre-compilation step failed (this is expected if dummy files can't be created/read)" @warn "Pre-compilation step failed (this is expected if dummy files can't be created/read)"
finally finally
@ -757,15 +801,18 @@ Loads the dataset registry from a JSON file.
- A dictionary containing the registry data. Returns an empty dictionary if the file doesn't exist or fails to parse. - A dictionary containing the registry data. Returns an empty dictionary if the file doesn't exist or fails to parse.
""" """
function load_registry(registry_path) function load_registry(registry_path)
if isfile(registry_path) return lock(REGISTRY_LOCK) do
try if isfile(registry_path)
return JSON.parsefile(registry_path, dicttype=Dict{String,Any}) try
catch e JSON.parsefile(registry_path, dicttype=Dict{String,Any})
@error "Failed to parse registry.json: $e" catch e
return Dict{String,Any}() @error "Failed to parse registry.json: $e"
Dict{String,Any}()
end
else
Dict{String,Any}()
end end
end end
return Dict{String,Any}()
end end
""" """
@ -839,32 +886,43 @@ Adds or updates an entry in the dataset registry JSON file.
- `is_imzML`: Boolean indicating if the source is an imzML file. - `is_imzML`: Boolean indicating if the source is an imzML file.
""" """
function update_registry(registry_path, dataset_name, source_path, metadata=nothing, is_imzML=false) function update_registry(registry_path, dataset_name, source_path, metadata=nothing, is_imzML=false)
registry = load_registry(registry_path) lock(REGISTRY_LOCK) do
registry = if isfile(registry_path)
# Get existing entry if it exists, otherwise create new one try
existing_entry = get(registry, dataset_name, Dict{String,Any}()) JSON.parsefile(registry_path, dicttype=Dict{String,Any})
catch e
# Start with existing data and update only the basic fields @error "Failed to parse registry.json while updating: $e"
entry = copy(existing_entry) Dict{String,Any}() # Start with empty if parsing fails
entry["source_path"] = source_path end
entry["processed_date"] = string(now()) else
entry["is_imzML"] = is_imzML Dict{String,Any}()
end
# Only update metadata if provided
if metadata !== nothing # Get existing entry if it exists, otherwise create new one
entry["metadata"] = metadata existing_entry = get(registry, dataset_name, Dict{String,Any}())
end
# Start with existing data and update only the basic fields
# Note: has_mask and mask_path are preserved from existing_entry if they exist entry = copy(existing_entry)
entry["source_path"] = source_path
registry[dataset_name] = entry entry["processed_date"] = string(now())
entry["is_imzML"] = is_imzML
try
open(registry_path, "w") do f # Only update metadata if provided
JSON.print(f, registry, 4) if metadata !== nothing
entry["metadata"] = metadata
end
# Note: has_mask and mask_path are preserved from existing_entry if they exist
registry[dataset_name] = entry
try
open(registry_path, "w") do f
JSON.print(f, registry, 4)
end
catch e
@error "Failed to write to registry.json: $e"
end end
catch e
@error "Failed to write to registry.json: $e"
end end
end end
@ -945,18 +1003,27 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov
if all(iszero, slice) if all(iszero, slice)
sliceQuant = zeros(UInt8, size(slice)) sliceQuant = zeros(UInt8, size(slice))
bounds = (0.0, 1.0) # Default bounds for empty slice
@warn "No intensity data for m/z = $mass in $(dataset_name)" @warn "No intensity data for m/z = $mass in $(dataset_name)"
else else
sliceQuant = params.triqE ? TrIQ(slice, params.colorL, params.triqP, mask_matrix=mask_matrix_for_triq) : quantize_intensity(slice, params.colorL, mask_matrix=mask_matrix_for_triq) # Get both quantized data AND bounds in one call
if params.triqE
sliceQuant, bounds = TrIQ(slice, params.colorL, params.triqP, mask_matrix=mask_matrix_for_triq)
else
sliceQuant, bounds = quantize_intensity(slice, params.colorL, mask_matrix=mask_matrix_for_triq)
end
if params.medianF if params.medianF
sliceQuant = round.(UInt8, median_filter(sliceQuant)) sliceQuant = round.(UInt8, median_filter(sliceQuant))
# Note: bounds remain the same after median filter
end end
end end
save_bitmap(joinpath(output_dir, bitmap_filename), sliceQuant, ViridisPalette) save_bitmap(joinpath(output_dir, bitmap_filename), sliceQuant, ViridisPalette)
if !all(iszero, slice) if !all(iszero, slice)
generate_colorbar_image(slice, params.colorL, joinpath(output_dir, colorbar_filename); # Now pass the precomputed bounds to colorbar generation
use_triq=params.triqE, triq_prob=params.triqP, mask_path=mask_path) generate_colorbar_image(slice, params.colorL, joinpath(output_dir, colorbar_filename),
bounds; use_triq=params.triqE, triq_prob=params.triqP, mask_path=mask_path)
end end
end end
@ -980,28 +1047,39 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov
end end
function update_registry_mask_fields(registry_path, dataset_name, has_mask, mask_path) function update_registry_mask_fields(registry_path, dataset_name, has_mask, mask_path)
registry = load_registry(registry_path) lock(REGISTRY_LOCK) do
registry = if isfile(registry_path)
if haskey(registry, dataset_name) try
registry[dataset_name]["has_mask"] = has_mask JSON.parsefile(registry_path, dicttype=Dict{String,Any})
registry[dataset_name]["mask_path"] = mask_path catch e
else @error "Failed to parse registry.json while updating mask fields: $e"
# Create new entry if it doesn't exist Dict{String,Any}()
registry[dataset_name] = Dict{String,Any}( end
"has_mask" => has_mask, else
"mask_path" => mask_path, Dict{String,Any}()
"source_path" => "", end
"processed_date" => string(now()),
"is_imzML" => false if haskey(registry, dataset_name)
) registry[dataset_name]["has_mask"] = has_mask
end registry[dataset_name]["mask_path"] = mask_path
else
try # Create new entry if it doesn't exist
open(registry_path, "w") do f registry[dataset_name] = Dict{String,Any}(
JSON.print(f, registry, 4) "has_mask" => has_mask,
"mask_path" => mask_path,
"source_path" => "",
"processed_date" => string(now()),
"is_imzML" => false
)
end
try
open(registry_path, "w") do f
JSON.print(f, registry, 4)
end
catch e
@error "Failed to write to registry.json: $e"
end end
catch e
@error "Failed to write to registry.json: $e"
end end
end end

98
mask.jl
View File

@ -713,59 +713,57 @@ end
@onchange isready begin @onchange isready begin
if isready && !registry_init_done if isready && !registry_init_done
@async begin # Run asynchronously to not block startup sleep(1.0) # Give frontend time to initialize
sleep(1.0) # Give frontend time to initialize try
try println("Synchronizing registry for mask editor...")
println("Synchronizing registry for mask editor...") reg_path = abspath(joinpath(@__DIR__, "public", "registry.json"))
reg_path = abspath(joinpath(@__DIR__, "public", "registry.json")) # Assuming load_registry is available from MSI_src or a similar utility file
# Assuming load_registry is available from MSI_src or a similar utility file # For now, handle its absence gracefully if it's not explicitly defined here.
# For now, handle its absence gracefully if it's not explicitly defined here. registry = isfile(reg_path) ? load_registry(reg_path) : Dict{String, Any}()
registry = isfile(reg_path) ? load_registry(reg_path) : Dict{String, Any}()
public_dirs = isdir("public") ? readdir("public") : [] public_dirs = isdir("public") ? readdir("public") : []
ignored_dirs = ["css", "masks"] ignored_dirs = ["css", "masks"]
dataset_dirs = filter(d -> isdir(joinpath("public", d)) && !(d in ignored_dirs), public_dirs) dataset_dirs = filter(d -> isdir(joinpath("public", d)) && !(d in ignored_dirs), public_dirs)
registry_keys = Set(keys(registry)) registry_keys = Set(keys(registry))
folder_set = Set(dataset_dirs) folder_set = Set(dataset_dirs)
new_folders = setdiff(folder_set, registry_keys) new_folders = setdiff(folder_set, registry_keys)
for folder in new_folders for folder in new_folders
println("Found new folder: $folder") println("Found new folder: $folder")
registry[folder] = Dict( registry[folder] = Dict(
"source_path" => "unknown (manually added)", "source_path" => "unknown (manually added)",
"processed_date" => "unknown", "processed_date" => "unknown",
"metadata" => Dict(), "metadata" => Dict(),
"is_imzML" => true # Assume folder contains images if found this way "is_imzML" => true # Assume folder contains images if found this way
) )
end
removed_folders = setdiff(registry_keys, folder_set)
for folder in removed_folders
delete!(registry, folder)
end
if !isempty(new_folders) || !isempty(removed_folders)
println("Registry changed, saving...")
open(reg_path, "w") do f
JSON.print(f, registry, 4)
end
end
all_folders = sort(collect(keys(registry)), lt=natural)
img_folders = filter(folder -> get(get(registry, folder, Dict()), "is_imzML", false), all_folders)
available_folders = deepcopy(all_folders)
image_available_folders = deepcopy(img_folders)
println("Mask editor UI lists updated. All: $(length(available_folders)), Images: $(length(image_available_folders))")
catch e
@warn "Mask editor registry synchronization failed: $e"
available_folders = []
image_available_folders = []
finally
registry_init_done = true
end end
removed_folders = setdiff(registry_keys, folder_set)
for folder in removed_folders
delete!(registry, folder)
end
if !isempty(new_folders) || !isempty(removed_folders)
println("Registry changed, saving...")
open(reg_path, "w") do f
JSON.print(f, registry, 4)
end
end
all_folders = sort(collect(keys(registry)), lt=natural)
img_folders = filter(folder -> get(get(registry, folder, Dict()), "is_imzML", false), all_folders)
available_folders = deepcopy(all_folders)
image_available_folders = deepcopy(img_folders)
println("Mask editor UI lists updated. All: $(length(available_folders)), Images: $(length(image_available_folders))")
catch e
@warn "Mask editor registry synchronization failed: $e"
available_folders = []
image_available_folders = []
finally
registry_init_done = true
end end
end end
end end

50
src/Common.jl Normal file
View File

@ -0,0 +1,50 @@
# src/Common.jl
# --- Unified Error Types ---
abstract type MSIError <: Exception end
struct FileFormatError <: MSIError
msg::String
end
struct SpectrumNotFoundError <: MSIError
id
end
struct InvalidSpectrumError <: MSIError
id
reason::String
end
# --- Shared Constants ---
const DEFAULT_CACHE_SIZE = 100
const DEFAULT_NUM_BINS = 2000
"""
validate_spectrum_data(mz, intensity, id)
Checks a spectrum for common data integrity issues.
- Checks for `NaN` or `Inf` values in intensity and m/z arrays.
- Verifies that the m/z array is sorted in ascending order.
- Ensures that m/z and intensity arrays have the same length.
# Throws
- `InvalidSpectrumError` if any of the checks fail.
"""
function validate_spectrum_data(mz::AbstractVector, intensity::AbstractVector, id)
if length(mz) != length(intensity)
throw(InvalidSpectrumError(id, "m/z and intensity arrays have different lengths"))
end
if !all(isfinite, mz) || !all(isfinite, intensity)
throw(InvalidSpectrumError(id, "Spectrum contains NaN or Inf values"))
end
if !issorted(mz)
throw(InvalidSpectrumError(id, "m/z array is not sorted"))
end
return true
end

231
src/MSIData.jl
View File

@ -46,9 +46,10 @@ Contains metadata for a single binary data array (m/z or intensity) within a spe
- `format`: The data type of the elements (e.g., `Float32`, `Int64`). - `format`: The data type of the elements (e.g., `Float32`, `Int64`).
- `is_compressed`: A boolean flag indicating if the data is compressed (e.g., with zlib). - `is_compressed`: A boolean flag indicating if the data is compressed (e.g., with zlib).
- `offset`: The byte offset of the data within the file (`.ibd` for imzML, `.mzML` for mzML). - `offset`: The byte offset of the data within the file (`.ibd` for imzML, `.mzML` for mzML).
- `encoded_length`: The length of the data. For uncompressed imzML, this is the number of - `encoded_length`: The length of the data, which has different meanings depending on the format:
elements in the array. For compressed imzML, it is the number of bytes of the compressed - **Uncompressed imzML**: The number of elements (points) in the array.
data. For mzML, this is the length of the Base64 encoded string. - **Compressed imzML**: The number of bytes of the compressed data chunk in the `.ibd` file.
- **mzML**: The number of characters in the Base64 encoded string within the `.mzML` file.
- `axis_type`: A symbol (`:mz` or `:intensity`) indicating the type of data. - `axis_type`: A symbol (`:mz` or `:intensity`) indicating the type of data.
""" """
struct SpectrumAsset struct SpectrumAsset
@ -61,6 +62,15 @@ struct SpectrumAsset
axis_type::Symbol axis_type::Symbol
end end
"""
SpectrumMode
An enum representing the type of a spectrum's data.
- `CENTROID`: The spectrum contains centroided data (i.e., processed peaks).
- `PROFILE`: The spectrum contains raw profile data.
- `UNKNOWN`: The spectrum type is not specified.
"""
@enum SpectrumMode CENTROID=1 PROFILE=2 UNKNOWN=3 @enum SpectrumMode CENTROID=1 PROFILE=2 UNKNOWN=3
""" """
@ -116,7 +126,7 @@ mutable struct MSIData
coordinate_map::Union{Matrix{Int}, Nothing} # Maps (x,y) to linear index for imzML coordinate_map::Union{Matrix{Int}, Nothing} # Maps (x,y) to linear index for imzML
# LRU Cache for GetSpectrum # LRU Cache for GetSpectrum
cache::Dict{Int, Tuple{Vector, Vector}} cache::Dict{Int, Tuple{Vector{Float64}, Vector{Float64}}}
cache_order::Vector{Int} # Stores indices, with most recently used at the end cache_order::Vector{Int} # Stores indices, with most recently used at the end
cache_size::Int # Max number of spectra in cache cache_size::Int # Max number of spectra in cache
cache_lock::ReentrantLock # To make cache access thread-safe cache_lock::ReentrantLock # To make cache access thread-safe
@ -143,6 +153,74 @@ mutable struct MSIData
end end
end end
"""
Base.close(data::MSIData)
Explicitly closes the file handles associated with the `MSIData` object and clears the spectrum cache.
It is good practice to call this method when you are finished with an `MSIData` object to release resources immediately.
"""
function Base.close(data::MSIData)
if data.source isa ImzMLSource && isopen(data.source.ibd_handle)
close(data.source.ibd_handle)
elseif data.source isa MzMLSource && isopen(data.source.file_handle)
close(data.source.file_handle)
end
# Clear cache
empty!(data.cache)
empty!(data.cache_order)
println("MSIData object closed and resources released.")
end
"""
warm_cache(data::MSIData, indices::AbstractVector{Int})
Pre-loads a specified list of spectra into the cache. This is useful for warming up
the cache with frequently accessed spectra to improve performance for subsequent
interactive analysis.
"""
function warm_cache(data::MSIData, indices::AbstractVector{Int})
println("Warming cache with $(length(indices)) spectra...")
for idx in indices
# Calling GetSpectrum will load the data and place it in the cache
GetSpectrum(data, idx)
end
println("Cache warming complete.")
end
"""
get_memory_usage(data::MSIData)
Calculates and returns a summary of the memory currently used by the MSIData object,
including the spectrum cache and metadata.
# Returns
- A `NamedTuple` with memory usage in bytes for different components.
"""
function get_memory_usage(data::MSIData)
cache_bytes = 0
lock(data.cache_lock) do
# This is an approximation; `sizeof` on a dictionary is not fully representative,
# but it's much better to sum the size of the actual vectors.
for (mz, intensity) in values(data.cache)
cache_bytes += sizeof(mz) + sizeof(intensity)
end
end
metadata_bytes = sizeof(data.spectra_metadata)
coordinate_map_bytes = data.coordinate_map === nothing ? 0 : sizeof(data.coordinate_map)
total_bytes = cache_bytes + metadata_bytes + coordinate_map_bytes
return (
total_mb = total_bytes / 1024^2,
cache_mb = cache_bytes / 1024^2,
metadata_mb = metadata_bytes / 1024^2,
coordinate_map_mb = coordinate_map_bytes / 1024^2
)
end
""" """
get_masked_spectrum_indices(data::MSIData, mask_matrix::BitMatrix) -> Set{Int} get_masked_spectrum_indices(data::MSIData, mask_matrix::BitMatrix) -> Set{Int}
@ -158,14 +236,14 @@ spectrum indices that fall within the `true` regions of the mask.
""" """
function get_masked_spectrum_indices(data::MSIData, mask_matrix::BitMatrix) function get_masked_spectrum_indices(data::MSIData, mask_matrix::BitMatrix)
if data.coordinate_map === nothing if data.coordinate_map === nothing
error("Coordinate map not available. Cannot apply mask to non-imaging data.") throw(ArgumentError("Coordinate map not available. Cannot apply mask to non-imaging data."))
end end
width, height = data.image_dims width, height = data.image_dims
mask_height, mask_width = size(mask_matrix) mask_height, mask_width = size(mask_matrix)
if mask_width != width || mask_height != height if mask_width != width || mask_height != height
error("Mask dimensions ($(mask_width)x$(mask_height)) do not match image dimensions ($(width)x$(height)).") throw(ArgumentError("Mask dimensions ($(mask_width)x$(mask_height)) do not match image dimensions ($(width)x$(height))."))
end end
masked_indices = Set{Int}() masked_indices = Set{Int}()
@ -209,6 +287,9 @@ by this function and is assumed to be handled by the caller if necessary.
- A `Vector` of the appropriate type containing the decoded data. - A `Vector` of the appropriate type containing the decoded data.
""" """
function read_binary_vector(io::IO, asset::SpectrumAsset) function read_binary_vector(io::IO, asset::SpectrumAsset)
if asset.offset < 0 || asset.offset >= filesize(io)
throw(FileFormatError("Invalid asset offset: $(asset.offset) for file size $(filesize(io))"))
end
seek(io, asset.offset) seek(io, asset.offset)
raw_b64 = read(io, asset.encoded_length) raw_b64 = read(io, asset.encoded_length)
decoded_bytes = base64decode(strip(String(raw_b64))) decoded_bytes = base64decode(strip(String(raw_b64)))
@ -217,6 +298,10 @@ function read_binary_vector(io::IO, asset::SpectrumAsset)
n_elements = Int(bytes_io.size / sizeof(asset.format)) n_elements = Int(bytes_io.size / sizeof(asset.format))
out_array = Array{asset.format}(undef, n_elements) out_array = Array{asset.format}(undef, n_elements)
read!(bytes_io, out_array) read!(bytes_io, out_array)
# FIX: little-endian to host byte order for mzML data
out_array .= ltoh.(out_array)
return out_array return out_array
end end
@ -241,17 +326,29 @@ function read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata)
mz = Array{source.mz_format}(undef, meta.mz_asset.encoded_length) mz = Array{source.mz_format}(undef, meta.mz_asset.encoded_length)
intensity = Array{source.intensity_format}(undef, meta.int_asset.encoded_length) intensity = Array{source.intensity_format}(undef, meta.int_asset.encoded_length)
seek(source.ibd_handle, meta.mz_asset.offset) # FIX: Read arrays in the order they appear in the file for efficiency,
read!(source.ibd_handle, mz) # but ensure we seek to the correct offset for both.
if meta.mz_asset.offset < meta.int_asset.offset
seek(source.ibd_handle, meta.int_asset.offset) seek(source.ibd_handle, meta.mz_asset.offset)
read!(source.ibd_handle, intensity) read!(source.ibd_handle, mz)
seek(source.ibd_handle, meta.int_asset.offset)
read!(source.ibd_handle, intensity)
else
seek(source.ibd_handle, meta.int_asset.offset)
read!(source.ibd_handle, intensity)
seek(source.ibd_handle, meta.mz_asset.offset)
read!(source.ibd_handle, mz)
end
# imzML data is little-endian. Convert to host byte order. # imzML data is little-endian. Convert to host byte order.
mz .= ltoh.(mz) mz .= ltoh.(mz)
intensity .= ltoh.(intensity) intensity .= ltoh.(intensity) # FIX: Added missing conversion for intensity
return mz, intensity mz_f64, intensity_f64 = Float64.(mz), Float64.(intensity)
validate_spectrum_data(mz_f64, intensity_f64, meta.id)
return mz_f64, intensity_f64
end end
""" """
@ -272,7 +369,12 @@ function read_spectrum_from_disk(source::MzMLSource, meta::SpectrumMetadata)
# For mzML, data is Base64 encoded within the XML # For mzML, data is Base64 encoded within the XML
mz = read_binary_vector(source.file_handle, meta.mz_asset) mz = read_binary_vector(source.file_handle, meta.mz_asset)
intensity = read_binary_vector(source.file_handle, meta.int_asset) intensity = read_binary_vector(source.file_handle, meta.int_asset)
return mz, intensity
mz_f64, intensity_f64 = Float64.(mz), Float64.(intensity)
validate_spectrum_data(mz_f64, intensity_f64, meta.id)
return mz_f64, intensity_f64
end end
# --- Public API --- # # --- Public API --- #
@ -294,7 +396,7 @@ This function is the core of the "Indexed" and "Cache" access patterns.
""" """
function GetSpectrum(data::MSIData, index::Int) function GetSpectrum(data::MSIData, index::Int)
if index < 1 || index > length(data.spectra_metadata) if index < 1 || index > length(data.spectra_metadata)
error("Spectrum index $index out of bounds.") throw(SpectrumNotFoundError(index))
end end
# Phase 1: Check the cache (with lock) # Phase 1: Check the cache (with lock)
@ -352,16 +454,16 @@ the indexed `GetSpectrum` method, benefiting from caching.
""" """
function GetSpectrum(data::MSIData, x::Int, y::Int) function GetSpectrum(data::MSIData, x::Int, y::Int)
if data.coordinate_map === nothing if data.coordinate_map === nothing
error("Coordinate map not available. This method is only for imaging data loaded from .imzML files.") throw(ArgumentError("Coordinate map not available. This method is only for imaging data loaded from .imzML files."))
end end
width, height = data.image_dims width, height = data.image_dims
if x < 1 || x > width || y < 1 || y > height if x < 1 || x > width || y < 1 || y > height
error("Coordinates ($x, $y) out of bounds for image dimensions ($width, $height).") throw(ArgumentError("Coordinates ($x, $y) out of bounds for image dimensions ($width, $height)."))
end end
index = data.coordinate_map[x, y] index = data.coordinate_map[x, y]
if index == 0 if index == 0
error("No spectrum found at coordinates ($x, $y).") throw(SpectrumNotFoundError((x, y)))
end end
return GetSpectrum(data, index) # Call the existing method return GetSpectrum(data, index) # Call the existing method
@ -373,13 +475,14 @@ end
A "function barrier" helper for safely processing a single spectrum. A "function barrier" helper for safely processing a single spectrum.
This function retrieves a spectrum and then immediately calls the provided function `f` This function retrieves a spectrum and then immediately calls the provided function `f`
with the resulting `(mz, intensity)` arrays. This pattern is crucial for performance. with the resulting `(mz, intensity)` arrays. While the `GetSpectrum` API is now type-stable,
While `GetSpectrum` itself is type-unstable (because the array data types are not this pattern remains good practice for separating data access from data processing.
known at compile time), calling `f` with the result allows Julia's compiler to
generate specialized, fast code for `f` based on the *concrete* types it receives.
Use this function when you need to perform performance-critical operations on a Use this function when you need to perform performance-critical operations on a
single spectrum. Your logic should be inside the function passed to this helper. single spectrum. Your logic should be inside the function passed to this helper.
For bulk processing of all spectra, consider using the `_iterate_spectra_fast`
function, which is optimized for sequential access and avoids cache overhead.
``` ```
""" """
function process_spectrum(f::Function, data::MSIData, index::Int) function process_spectrum(f::Function, data::MSIData, index::Int)
@ -611,19 +714,14 @@ function get_total_spectrum_imzml(msi_data::MSIData; num_bins::Int=2000, masked_
intensity_view = intensity intensity_view = intensity
# Manual binning without clamp/round for SIMD # Manual binning without clamp/round for SIMD
@inbounds for i in eachindex(mz_view) @inbounds @simd for i in eachindex(mz_view)
# Calculate raw bin index (much faster than round+clamp) # Calculate raw bin index (much faster than round+clamp)
raw_index = (mz_view[i] - min_mz) * inv_bin_step + 1.0 raw_index = (mz_view[i] - min_mz) * inv_bin_step + 1.0
bin_index = trunc(Int, raw_index) bin_index = trunc(Int, raw_index)
# Manual bounds checking (faster than clamp) # Branchless version using clamp
if 1 <= bin_index <= num_bins final_index = clamp(bin_index, 1, num_bins)
intensity_sum[bin_index] += intensity_view[i] intensity_sum[final_index] += intensity_view[i]
elseif bin_index < 1
intensity_sum[1] += intensity_view[i]
else # bin_index > num_bins
intensity_sum[num_bins] += intensity_view[i]
end
end end
end end
pass2_duration = (time_ns() - pass2_start_time) / 1e9 pass2_duration = (time_ns() - pass2_start_time) / 1e9
@ -706,19 +804,14 @@ function get_total_spectrum_mzml(msi_data::MSIData; num_bins::Int=2000, masked_i
return return
end end
@inbounds for i in eachindex(mz) @inbounds @simd for i in eachindex(mz)
# Calculate raw bin index # Calculate raw bin index
raw_index = (mz[i] - min_mz) * inv_bin_step + 1.0 raw_index = (mz[i] - min_mz) * inv_bin_step + 1.0
bin_index = trunc(Int, raw_index) bin_index = trunc(Int, raw_index)
# Manual bounds checking # Branchless version using clamp
if 1 <= bin_index <= num_bins final_index = clamp(bin_index, 1, num_bins)
intensity_sum[bin_index] += intensity[i] intensity_sum[final_index] += intensity[i]
elseif bin_index < 1
intensity_sum[1] += intensity[i]
else # bin_index > num_bins
intensity_sum[num_bins] += intensity[i]
end
end end
end end
pass2_duration = (time_ns() - pass2_start_time) / 1e9 pass2_duration = (time_ns() - pass2_start_time) / 1e9
@ -842,9 +935,14 @@ end
Reads a single data array (m/z or intensity) from an `.ibd` file stream, Reads a single data array (m/z or intensity) from an `.ibd` file stream,
handling both compressed and uncompressed data. handling both compressed and uncompressed data.
- If `asset.is_compressed` is true, it reads the compressed bytes, inflates This is an internal function designed for high-performance iteration. It assumes
them using zlib, and reinterprets the result as a vector of the given `format`. the file stream `io` is already positioned at the correct offset.
- If false, it reads the uncompressed data directly into a vector.
- If `asset.is_compressed` is true, it reads `asset.encoded_length` bytes of
compressed data, inflates them using zlib, and reinterprets the result as a
vector of the given `format`.
- If false, it reads `asset.encoded_length` *elements* of uncompressed data
directly into a vector.
# Arguments # Arguments
- `io`: The IO stream of the `.ibd` file. - `io`: The IO stream of the `.ibd` file.
@ -853,8 +951,16 @@ handling both compressed and uncompressed data.
# Returns # Returns
- A `Vector` containing the data. - A `Vector` containing the data.
# Throws
- An error if zlib decompression fails, which can indicate corrupt data or
an incorrect offset in the `.imzML` metadata.
""" """
function read_compressed_array(io::IO, asset::SpectrumAsset, format::Type) function read_compressed_array(io::IO, asset::SpectrumAsset, format::Type)
# Add validation before seeking
if asset.offset < 0 || asset.offset >= filesize(io)
throw(FileFormatError("Invalid asset offset: $(asset.offset) for file size $(filesize(io))"))
end
seek(io, asset.offset) seek(io, asset.offset)
if asset.is_compressed if asset.is_compressed
@ -930,10 +1036,12 @@ function _iterate_uncompressed_fast(f::Function, data::MSIData, source::ImzMLSou
if meta.mz_asset.offset < meta.int_asset.offset if meta.mz_asset.offset < meta.int_asset.offset
seek(source.ibd_handle, meta.mz_asset.offset) seek(source.ibd_handle, meta.mz_asset.offset)
read!(source.ibd_handle, mz_view) read!(source.ibd_handle, mz_view)
seek(source.ibd_handle, meta.int_asset.offset) # FIX: Added missing seek
read!(source.ibd_handle, int_view) read!(source.ibd_handle, int_view)
else else
seek(source.ibd_handle, meta.int_asset.offset) seek(source.ibd_handle, meta.int_asset.offset)
read!(source.ibd_handle, int_view) read!(source.ibd_handle, int_view)
seek(source.ibd_handle, meta.mz_asset.offset) # FIX: Added missing seek
read!(source.ibd_handle, mz_view) read!(source.ibd_handle, mz_view)
end end
@ -1020,15 +1128,19 @@ provided function. It bypasses the `GetSpectrum` cache to avoid storing
all decoded spectra in memory. all decoded spectra in memory.
""" """
function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::MzMLSource, indices_to_iterate::Union{AbstractVector{Int}, Nothing}) function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::MzMLSource, indices_to_iterate::Union{AbstractVector{Int}, Nothing})
# This implementation is for mzML. It iterates through each spectrum, # This implementation is for mzML. To improve disk I/O, we can reorder the read
# decodes it, and then calls the function `f`. It's less performant than # operations to be as sequential as possible based on their offset in the file.
# the imzML version because we cannot pre-allocate buffers of a known size,
# but it is correct and still faster than using GetSpectrum due to bypassing the cache.
# Determine which indices to iterate over # Determine which indices to iterate over
spectrum_indices = (indices_to_iterate === nothing) ? (1:length(data.spectra_metadata)) : indices_to_iterate spectrum_indices = (indices_to_iterate === nothing) ? (1:length(data.spectra_metadata)) : indices_to_iterate
for i in spectrum_indices # Create a vector of (index, offset) tuples to be sorted
indices_with_offsets = [(i, data.spectra_metadata[i].mz_asset.offset) for i in spectrum_indices]
# Sort by offset to make disk access more sequential
sort!(indices_with_offsets, by = x -> x[2])
for (i, _) in indices_with_offsets
meta = data.spectra_metadata[i] meta = data.spectra_metadata[i]
mz = read_binary_vector(source.file_handle, meta.mz_asset) mz = read_binary_vector(source.file_handle, meta.mz_asset)
@ -1039,14 +1151,27 @@ function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::MzMLSour
end end
""" """
_iterate_spectra_fast(f::Function, data::MSIData) _iterate_spectra_fast(f::Function, data::MSIData, indices_to_iterate=nothing)
An internal, high-performance iterator for bulk processing that bypasses the cache. An internal, high-performance iterator for bulk processing that bypasses the `GetSpectrum` cache.
It dispatches to a specialized implementation based on the data source type It provides direct, sequential access to spectral data, making it the most efficient
(`ImzMLSource` or `MzMLSource`). method for operations that need to process many or all spectra (e.g., calculating
a total spectrum, pre-computing analytics, or exporting data).
- `f`: A function to call for each spectrum, with signature `f(index, mz, intensity)`. This function dispatches to a specialized implementation based on the data source type
(`ImzMLSource` or `MzMLSource`) and data characteristics (e.g., compressed vs.
uncompressed) to maximize performance.
# Arguments
- `f`: A function to call for each spectrum, with the signature `f(index, mz, intensity)`.
- `data`: The `MSIData` object. - `data`: The `MSIData` object.
- `indices_to_iterate`: An optional `AbstractVector{Int}` specifying a subset of
spectrum indices to iterate over. If `nothing`, it iterates over all spectra.
# Warning
This is a low-level function that bypasses the public `GetSpectrum` API. It does not
use the cache and is not guaranteed to be thread-safe if the same `MSIData` object
is accessed from multiple threads concurrently.
""" """
function _iterate_spectra_fast(f::Function, data::MSIData, indices_to_iterate::Union{AbstractVector{Int}, Nothing}=nothing) function _iterate_spectra_fast(f::Function, data::MSIData, indices_to_iterate::Union{AbstractVector{Int}, Nothing}=nothing)
# Dispatch to the correct implementation based on the source type # Dispatch to the correct implementation based on the source type

1
src/MSI_src.jl
View File

@ -40,6 +40,7 @@ export FeatureMatrix,
get_common_calibration_standards get_common_calibration_standards
# Include all source files directly into the main module # Include all source files directly into the main module
include("Common.jl")
include("MSIData.jl") include("MSIData.jl")
include("ParserHelpers.jl") include("ParserHelpers.jl")
include("mzML.jl") include("mzML.jl")

197
src/MzmlConverter.jl
View File

@ -5,7 +5,7 @@ into a proper .imzML/.ibd file pair, using a separate synchronization file.
It replicates the functionality of the original R scripts that use MALDIquant. It replicates the functionality of the original R scripts that use MALDIquant.
""" """
using DataFrames, Printf, CSV using DataFrames, Printf, CSV, UUIDs, ProgressMeter
# This file assumes that the main application file (e.g., app.jl) has already included # This file assumes that the main application file (e.g., app.jl) has already included
# the necessary source files: MSIData.jl, mzML.jl, imzML.jl # the necessary source files: MSIData.jl, mzML.jl, imzML.jl
@ -396,6 +396,7 @@ This function handles three cases:
- A tuple `(mz_array, intensity_array)` for the rendered pixel spectrum. - A tuple `(mz_array, intensity_array)` for the rendered pixel spectrum.
""" """
function RenderPixel( function RenderPixel(
intensity_buffer::Vector{Float32},
pixel_info::AbstractVector{Int64}, pixel_info::AbstractVector{Int64},
scans::AbstractMatrix{Int64}, scans::AbstractMatrix{Int64},
msi_data::MSIData, msi_data::MSIData,
@ -408,19 +409,32 @@ function RenderPixel(
num_actions = last_scan - first_scan num_actions = last_scan - first_scan
# Get reference m/z array from the first scan involved. # Get reference m/z array from the first scan involved.
# This call remains type-unstable, but its impact is now isolated and only paid once.
mz_array, _ = GetSpectrum(msi_data, first_scan) mz_array, _ = GetSpectrum(msi_data, first_scan)
# If the first spectrum was empty, we can't do anything else. # If the first spectrum was empty, we can't do anything else.
if isempty(mz_array) if isempty(mz_array)
return (mz_array, Float32[]) return (mz_array, view(intensity_buffer, 0:-1), UNKNOWN)
end end
new_intensity = zeros(Float32, length(mz_array)) # Determine the mode for the output spectrum. If any contributing scan is profile, the result is profile.
final_mode = CENTROID
for i in first_scan:last_scan
if msi_data.spectra_metadata[i].mode == PROFILE
final_mode = PROFILE
break
end
end
# Use the provided buffer instead of allocating a new one.
if length(intensity_buffer) < length(mz_array)
error("Provided intensity buffer is too small for spectrum of length $(length(mz_array))")
end
new_intensity = view(intensity_buffer, 1:length(mz_array))
fill!(new_intensity, 0.0f0)
# SAFETY: Ensure we have valid scan indices # SAFETY: Ensure we have valid scan indices
if first_scan < 1 || last_scan > size(scans, 1) || first_scan > last_scan if first_scan < 1 || last_scan > size(scans, 1) || first_scan > last_scan
return (mz_array, new_intensity) # Return zero intensity for invalid ranges return (mz_array, new_intensity, final_mode) # Return zero intensity for invalid ranges
end end
if num_actions == 0 # Single scan contributes to the pixel if num_actions == 0 # Single scan contributes to the pixel
@ -469,7 +483,7 @@ function RenderPixel(
# FINAL SAFETY: Clamp any negative values to zero # FINAL SAFETY: Clamp any negative values to zero
new_intensity = max.(new_intensity, 0.0f0) new_intensity = max.(new_intensity, 0.0f0)
return (mz_array, new_intensity) return (mz_array, new_intensity, final_mode)
end end
""" """
@ -498,84 +512,96 @@ function ConvertMzmlToImzml(source_file::String, target_ibd_file::String, timing
open(target_ibd_file, "w") do ibd_stream open(target_ibd_file, "w") do ibd_stream
write(ibd_stream, zeros(UInt8, 16)) # UUID placeholder write(ibd_stream, zeros(UInt8, 16)) # UUID placeholder
end end
return BinaryMetadata[], Tuple{Int, Int}[], (0, 0), UNKNOWN return BinaryMetadata[], Tuple{Int, Int}[], (0, 0), SpectrumMode[], uuid4()
end end
width = maximum(timing_matrix[:, 1]) width = maximum(timing_matrix[:, 1])
height = maximum(timing_matrix[:, 2]) height = maximum(timing_matrix[:, 2])
msi_data = OpenMSIData(source_file) msi_data = OpenMSIData(source_file)
local binary_meta_vec, coords_vec, pixel_modes, ibd_uuid
try
precompute_analytics(msi_data)
max_points = isempty(msi_data.spectrum_stats_df.NumPoints) ? 0 : maximum(msi_data.spectrum_stats_df.NumPoints)
intensity_buffer = zeros(Float32, max_points)
source_mode = UNKNOWN scan_time_deltas = zeros(Int64, size(scans, 1))
if !isempty(msi_data.spectra_metadata) if size(scans, 1) > 1
source_mode = msi_data.spectra_metadata[1].mode for i in 1:(size(scans, 1) - 1)
end delta = scans[i+1, 2] - scans[i, 2]
scan_time_deltas[i] = max(1, delta)
scan_time_deltas = zeros(Int64, size(scans, 1))
if size(scans, 1) > 1
for i in 1:(size(scans, 1) - 1)
delta = scans[i+1, 2] - scans[i, 2]
scan_time_deltas[i] = max(1, delta)
end
scan_time_deltas[end] = max(1, scan_time_deltas[end-1])
end
pixel_time_deltas = zeros(Int64, size(timing_matrix, 1))
if size(timing_matrix, 1) > 1
for i in 1:(size(timing_matrix, 1) - 1)
delta = timing_matrix[i+1, 3] - timing_matrix[i, 3]
pixel_time_deltas[i] = max(1, delta)
end
pixel_time_deltas[end] = max(1, pixel_time_deltas[end-1])
end
binary_meta_vec = BinaryMetadata[]
sizehint!(binary_meta_vec, size(timing_matrix, 1))
coords_vec = Tuple{Int, Int}[]
sizehint!(coords_vec, size(timing_matrix, 1))
empty_pixel_count = 0
open(target_ibd_file, "w") do ibd_stream
write(ibd_stream, zeros(UInt8, 16)) # UUID placeholder
for i in 1:size(timing_matrix, 1)
pixel_info = timing_matrix[i, :]
x, y = pixel_info[1], pixel_info[2]
push!(coords_vec, (x, y))
first_scan = pixel_info[4]
last_scan = pixel_info[5]
if first_scan > last_scan || first_scan < 1 || last_scan > size(scans, 1)
empty_pixel_count += 1
# For empty pixels, offsets point to the current end of file, with zero length
current_pos = position(ibd_stream)
push!(binary_meta_vec, BinaryMetadata(current_pos, 0, current_pos, 0))
continue
end end
scan_time_deltas[end] = max(1, scan_time_deltas[end-1])
mz, intensity = RenderPixel(pixel_info, scans, msi_data, scan_time_deltas, pixel_time_deltas)
# Write m/z array
mz_offset = position(ibd_stream)
for val in mz
write(ibd_stream, htol(Float32(val)))
end
mz_length = position(ibd_stream) - mz_offset
# Write intensity array
int_offset = position(ibd_stream)
for val in intensity
write(ibd_stream, htol(Float32(val)))
end
int_length = position(ibd_stream) - int_offset
push!(binary_meta_vec, BinaryMetadata(mz_offset, mz_length, int_offset, int_length))
end end
end
@info "Found and processed $empty_pixel_count empty pixels out of $(size(timing_matrix, 1)) total." pixel_time_deltas = zeros(Int64, size(timing_matrix, 1))
return binary_meta_vec, coords_vec, (width, height), source_mode if size(timing_matrix, 1) > 1
for i in 1:(size(timing_matrix, 1) - 1)
delta = timing_matrix[i+1, 3] - timing_matrix[i, 3]
pixel_time_deltas[i] = max(1, delta)
end
pixel_time_deltas[end] = max(1, pixel_time_deltas[end-1])
end
binary_meta_vec = BinaryMetadata[]
sizehint!(binary_meta_vec, size(timing_matrix, 1))
coords_vec = Tuple{Int, Int}[]
sizehint!(coords_vec, size(timing_matrix, 1))
pixel_modes = SpectrumMode[]
sizehint!(pixel_modes, size(timing_matrix, 1))
empty_pixel_count = 0
open(target_ibd_file, "w") do ibd_stream
# Generate and write a valid UUID
ibd_uuid = uuid4()
write(ibd_stream, htol(ibd_uuid.value))
p = Progress(size(timing_matrix, 1), 1, "Converting pixels... ")
for i in 1:size(timing_matrix, 1)
pixel_info = timing_matrix[i, :]
x, y = pixel_info[1], pixel_info[2]
push!(coords_vec, (x, y))
first_scan = pixel_info[4]
last_scan = pixel_info[5]
if first_scan > last_scan || first_scan < 1 || last_scan > size(scans, 1)
empty_pixel_count += 1
current_pos = position(ibd_stream)
push!(binary_meta_vec, BinaryMetadata(current_pos, 0, current_pos, 0))
push!(pixel_modes, UNKNOWN) # Mode for empty pixel
next!(p)
continue
end
mz, intensity, pixel_mode = RenderPixel(intensity_buffer, pixel_info, scans, msi_data, scan_time_deltas, pixel_time_deltas)
push!(pixel_modes, pixel_mode)
# Write m/z array (as Float64)
mz_offset = position(ibd_stream)
write(ibd_stream, htol.(mz)) # mz is Vector{Float64}
mz_length = position(ibd_stream) - mz_offset
# Write intensity array (as Float32)
int_offset = position(ibd_stream)
write(ibd_stream, htol.(intensity)) # intensity is Vector{Float32}
int_length = position(ibd_stream) - int_offset
push!(binary_meta_vec, BinaryMetadata(mz_offset, mz_length, int_offset, int_length))
next!(p)
end
end
@info "Found and processed $empty_pixel_count empty pixels out of $(size(timing_matrix, 1)) total."
finally
close(msi_data)
end
return binary_meta_vec, coords_vec, (width, height), pixel_modes, ibd_uuid
end end
""" """
@ -595,7 +621,7 @@ experiment and data format.
# Returns # Returns
- `true` on success, `false` on failure. - `true` on success, `false` on failure.
""" """
function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, coords::Vector{Tuple{Int, Int}}, dims::Tuple{Int, Int}, mode::SpectrumMode) function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, coords::Vector{Tuple{Int, Int}}, dims::Tuple{Int, Int}, modes::Vector{SpectrumMode}, ibd_uuid::UUID)
ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") ibd_file = replace(target_file, r"\.imzML$"i => ".ibd")
if isempty(binary_meta) if isempty(binary_meta)
@ -625,6 +651,7 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
<cvParam cvRef="MS" accession="MS:1000579" name="MS1 spectrum"/> <cvParam cvRef="MS" accession="MS:1000579" name="MS1 spectrum"/>
<cvParam cvRef="IMS" accession="IMS:1000080" name="mass spectrum"/> <cvParam cvRef="IMS" accession="IMS:1000080" name="mass spectrum"/>
<cvParam cvRef="IMS" accession="IMS:1000031" name="processed"/> <cvParam cvRef="IMS" accession="IMS:1000031" name="processed"/>
<cvParam cvRef="IMS" accession="IMS:1000081" name="ibd uuid" value="$(ibd_uuid)"/>
</fileContent> </fileContent>
</fileDescription> </fileDescription>
""") """)
@ -632,7 +659,7 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
<referenceableParamGroup id="mzArray"> <referenceableParamGroup id="mzArray">
<cvParam cvRef="MS" accession="MS:1000576" name="no compression"/> <cvParam cvRef="MS" accession="MS:1000576" name="no compression"/>
<cvParam cvRef="MS" accession="MS:1000514" name="m/z array" unitCvRef="MS" unitAccession="MS:1000040" unitName="m/z"/> <cvParam cvRef="MS" accession="MS:1000514" name="m/z array" unitCvRef="MS" unitAccession="MS:1000040" unitName="m/z"/>
<cvParam cvRef="MS" accession="MS:1000521" name="32-bit float"/> <cvParam cvRef="MS" accession="MS:1000523" name="64-bit float"/>
</referenceableParamGroup> </referenceableParamGroup>
<referenceableParamGroup id="intensityArray"> <referenceableParamGroup id="intensityArray">
<cvParam cvRef="MS" accession="MS:1000576" name="no compression"/> <cvParam cvRef="MS" accession="MS:1000576" name="no compression"/>
@ -659,6 +686,8 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
<cvParam cvRef="IMS" accession="IMS:1000043" name="max count of pixel y" value="$(dims[2])"/> <cvParam cvRef="IMS" accession="IMS:1000043" name="max count of pixel y" value="$(dims[2])"/>
<cvParam cvRef="IMS" accession="IMS:1000046" name="pixel size x" value="1" unitCvRef="UO" unitAccession="UO:0000017" unitName="micrometer"/> <cvParam cvRef="IMS" accession="IMS:1000046" name="pixel size x" value="1" unitCvRef="UO" unitAccession="UO:0000017" unitName="micrometer"/>
<cvParam cvRef="IMS" accession="IMS:1000047" name="pixel size y" value="1" unitCvRef="UO" unitAccession="UO:0000017" unitName="micrometer"/> <cvParam cvRef="IMS" accession="IMS:1000047" name="pixel size y" value="1" unitCvRef="UO" unitAccession="UO:0000017" unitName="micrometer"/>
<cvParam cvRef="IMS" accession="IMS:1000092" name="line scan sequence" value="line scan"/>
<cvParam cvRef="IMS" accession="IMS:1000093" name="spotsize" value="1"/>
</scanSettings> </scanSettings>
</scanSettingsList> </scanSettingsList>
""") """)
@ -702,16 +731,16 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
push!(spectrum_offsets, spectrum_start) push!(spectrum_offsets, spectrum_start)
# Calculate number of points from byte length # Calculate number of points from byte length
mz_points = meta.mz_length ÷ sizeof(Float32) mz_points = meta.mz_length ÷ sizeof(Float64)
int_points = meta.int_length ÷ sizeof(Float32) int_points = meta.int_length ÷ sizeof(Float32)
write(imzml_stream, """ <spectrum id="Scan=$(i)" defaultArrayLength="$(mz_points)" index="$(i-1)"> write(imzml_stream, """ <spectrum id="Scan=$(i)" defaultArrayLength="$(mz_points)" index="$(i-1)">
<cvParam cvRef="MS" accession="MS:1000511" name="ms level" value="1"/> <cvParam cvRef="MS" accession="MS:1000511" name="ms level" value="1"/>
""") """)
if mode == CENTROID if modes[i] == CENTROID
write(imzml_stream, """ <cvParam cvRef="MS" accession="MS:1000127" name="centroid spectrum"/> write(imzml_stream, """ <cvParam cvRef="MS" accession="MS:1000127" name="centroid spectrum"/>
""") """)
else elseif modes[i] == PROFILE
write(imzml_stream, """ <cvParam cvRef="MS" accession="MS:1000128" name="profile spectrum"/> write(imzml_stream, """ <cvParam cvRef="MS" accession="MS:1000128" name="profile spectrum"/>
""") """)
end end
@ -771,6 +800,10 @@ Main workflow function to convert a .mzML file to an .imzML file.
* `img_height`: height dimention for the creation of the y axis * `img_height`: height dimention for the creation of the y axis
""" """
function ImportMzmlFile(source_file::String, sync_file::String, target_file::String; img_width::Int=0, img_height::Int=0) function ImportMzmlFile(source_file::String, sync_file::String, target_file::String; img_width::Int=0, img_height::Int=0)
if !isfile(source_file)
throw(ArgumentError("Source mzML file not found: $source_file"))
end
println("Step 1: Getting scan times from .mzML file...") println("Step 1: Getting scan times from .mzML file...")
scans = GetMzmlScanTime(source_file) scans = GetMzmlScanTime(source_file)
@ -779,13 +812,13 @@ function ImportMzmlFile(source_file::String, sync_file::String, target_file::Str
println("Step 3: Converting spectra and writing .ibd file...") println("Step 3: Converting spectra and writing .ibd file...")
ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") ibd_file = replace(target_file, r"\.imzML$"i => ".ibd")
binary_meta, coords, (width, height), source_mode = ConvertMzmlToImzml(source_file, ibd_file, timing_matrix, scans) binary_meta, coords, (width, height), pixel_modes, ibd_uuid = ConvertMzmlToImzml(source_file, ibd_file, timing_matrix, scans)
# Flip image vertically to match R script output # Flip image vertically to match R script output
flipped_coords = [(x, height - y + 1) for (x, y) in coords] flipped_coords = [(x, height - y + 1) for (x, y) in coords]
println("Step 4: Exporting .imzML metadata file...") println("Step 4: Exporting .imzML metadata file...")
success = ExportImzml(target_file, binary_meta, flipped_coords, (width, height), source_mode) success = ExportImzml(target_file, binary_meta, flipped_coords, (width, height), pixel_modes, ibd_uuid)
if success if success
println("Conversion successful: $target_file") println("Conversion successful: $target_file")

275
src/imzML.jl
View File

@ -1,11 +1,8 @@
# src/imzML.jl # src/imzML.jl
using Images, Statistics, CairoMakie, DataFrames, Printf, ColorSchemes, StatsBase using Images, Statistics, CairoMakie, DataFrames, Printf, ColorSchemes, StatsBase
# --- Extracted from imzML.jl ---
""" """
This file provides a library for parsing `.imzML` and `.ibd` files in pure Julia. This file provides a library for parsing `.imzML` and `.ibd` files in pure Julia.
It is intended to be included by a parent script.
Core Functions: Core Functions:
- `load_imzml_lazy`: The main function that orchestrates the parsing. - `load_imzml_lazy`: The main function that orchestrates the parsing.
@ -146,6 +143,37 @@ function get_spectrum_attributes(stream::IO, hIbd::IO)
return skip return skip
end end
function get_spectrum_attributes_v2(stream::IO, hIbd::IO)
# Look for position x
readuntil(stream, "IMS:1000050")
x_skip = 0
# Look for position y
readuntil(stream, "IMS:1000051")
y_skip = 0
# Determine order of mz/intensity arrays
pos_before = position(stream)
# Look for first external offset (could be mz or intensity)
readuntil(stream, "external offset")
current_line = readline(stream)
# Check which array comes first by looking at the param group reference
mz_first = occursin("mzArray", current_line) ? 3 : 4
# Find array length - look for the first external array length after coordinates
seek(stream, pos_before)
readuntil(stream, "IMS:1000103")
array_len_skip = 0
# Find spectrum end
readuntil(stream, "</spectrum>")
spectrum_end_skip = 0
return [x_skip, y_skip, mz_first, array_len_skip, spectrum_end_skip]
end
function determine_parser(stream::IO, mz_is_compressed::Bool, int_is_compressed::Bool) function determine_parser(stream::IO, mz_is_compressed::Bool, int_is_compressed::Bool)
start_pos = position(stream) start_pos = position(stream)
spectrum_xml = "" spectrum_xml = ""
@ -200,13 +228,13 @@ end
function load_imzml_lazy(file_path::String; cache_size::Int=100) function load_imzml_lazy(file_path::String; cache_size::Int=100)
println("DEBUG: Checking for .imzML file at $file_path") println("DEBUG: Checking for .imzML file at $file_path")
if !isfile(file_path) if !isfile(file_path)
error("Provided path is not a file: $(file_path)") throw(FileFormatError("Provided path is not a file: $(file_path)"))
end end
ibd_path = replace(file_path, r"\.(imzML|mzML)"i => ".ibd") ibd_path = replace(file_path, r"\.(imzML|mzML)"i => ".ibd")
println("DEBUG: Checking for .ibd file at $ibd_path") println("DEBUG: Checking for .ibd file at $ibd_path")
if !isfile(ibd_path) if !isfile(ibd_path)
error("Corresponding .ibd file not found for: $(file_path)") throw(FileFormatError("Corresponding .ibd file not found for: $(file_path)"))
end end
println("DEBUG: Opening file streams for .imzML and .ibd") println("DEBUG: Opening file streams for .imzML and .ibd")
@ -262,6 +290,7 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
spectra_metadata = parse_neofx(stream, hIbd, param_groups, width, height, num_spectra, spectra_metadata = parse_neofx(stream, hIbd, param_groups, width, height, num_spectra,
default_mz_format, default_intensity_format, default_mz_format, default_intensity_format,
mz_is_compressed, int_is_compressed, global_mode) mz_is_compressed, int_is_compressed, global_mode)
#=
elseif parser_type == :compressed elseif parser_type == :compressed
println("DEBUG: Using compressed parser.") println("DEBUG: Using compressed parser.")
spectra_metadata = parse_compressed(stream, hIbd, param_groups, width, height, num_spectra, spectra_metadata = parse_compressed(stream, hIbd, param_groups, width, height, num_spectra,
@ -273,6 +302,13 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
default_mz_format, default_intensity_format, default_mz_format, default_intensity_format,
mz_is_compressed, int_is_compressed, global_mode) mz_is_compressed, int_is_compressed, global_mode)
end end
=#
else
println("DEBUG: Using compressed parser.")
spectra_metadata = parse_compressed(stream, hIbd, param_groups, width, height, num_spectra,
default_mz_format, default_intensity_format,
mz_is_compressed, int_is_compressed, global_mode)
end
println("DEBUG: Metadata parsing complete.") println("DEBUG: Metadata parsing complete.")
@ -306,37 +342,46 @@ function parse_uncompressed(stream::IO, hIbd::IO, param_groups::Dict{String, Spe
width::Int32, height::Int32, num_spectra::Int32, width::Int32, height::Int32, num_spectra::Int32,
mz_format::DataType, intensity_format::DataType, mz_format::DataType, intensity_format::DataType,
mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode) mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode)
# Your existing working skip-based parser
println("DEBUG: Learning file structure from first spectrum...") println("DEBUG: Learning file structure from first spectrum...")
start_of_spectra_xml = position(stream) start_of_spectra_xml = position(stream)
attr = get_spectrum_attributes(stream, hIbd)
# Skip UUID at beginning of IBD file (from converter)
seek(hIbd, 16)
current_ibd_offset = position(hIbd) current_ibd_offset = position(hIbd)
attr = get_spectrum_attributes_v2(stream, hIbd)
seek(stream, start_of_spectra_xml) seek(stream, start_of_spectra_xml)
println("DEBUG: Initial IBD offset: $current_ibd_offset") println("DEBUG: Initial IBD offset (after UUID): $current_ibd_offset")
spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra) spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra)
mz_is_first = attr[3] == 3 mz_is_first = attr[3] == 3
for k in 1:num_spectra for k in 1:num_spectra
# Store the start position of this spectrum for mode detection
spectrum_start_pos = position(stream) spectrum_start_pos = position(stream)
# Use skip values learned from the first spectrum # Find X coordinate
skip(stream, attr[5]) # Skip to X coordinate value line = readuntil(stream, "IMS:1000050")
if eof(stream)
error("Unexpected EOF while looking for X coordinate")
end
val_tag_x = find_tag(stream, r"value=\"(\d+)\"") val_tag_x = find_tag(stream, r"value=\"(\d+)\"")
x = parse(Int32, val_tag_x.captures[1]) x = parse(Int32, val_tag_x.captures[1])
skip(stream, attr[6]) # Skip to Y coordinate value # Find Y coordinate
line = readuntil(stream, "IMS:1000051")
val_tag_y = find_tag(stream, r"value=\"(\d+)\"") val_tag_y = find_tag(stream, r"value=\"(\d+)\"")
y = parse(Int32, val_tag_y.captures[1]) y = parse(Int32, val_tag_y.captures[1])
skip(stream, attr[7]) # Skip to array length value # Find array length - look for external array length
line = readuntil(stream, "IMS:1000103")
val_tag_len = find_tag(stream, r"value=\"(\d+)\"") val_tag_len = find_tag(stream, r"value=\"(\d+)\"")
nPoints = parse(Int32, val_tag_len.captures[1]) nPoints = parse(Int32, val_tag_len.captures[1])
# For uncompressed data, use simple calculation # CRITICAL FIX: Use the actual formats from the XML, not hardcoded values
mz_len_bytes = nPoints * sizeof(mz_format) # The converter writes Float64 for mz, Float32 for intensity
int_len_bytes = nPoints * sizeof(intensity_format) mz_len_bytes = nPoints * sizeof(Float64) # Converter uses Float64 for mz
int_len_bytes = nPoints * sizeof(Float32) # Converter uses Float32 for intensity
local mz_offset, int_offset local mz_offset, int_offset
if mz_is_first if mz_is_first
@ -347,7 +392,7 @@ function parse_uncompressed(stream::IO, hIbd::IO, param_groups::Dict{String, Spe
mz_offset = int_offset + int_len_bytes mz_offset = int_offset + int_len_bytes
end end
# Mode detection from spectrum XML # Mode detection
current_pos = position(stream) current_pos = position(stream)
seek(stream, spectrum_start_pos) seek(stream, spectrum_start_pos)
spectrum_buffer = IOBuffer() spectrum_buffer = IOBuffer()
@ -369,14 +414,16 @@ function parse_uncompressed(stream::IO, hIbd::IO, param_groups::Dict{String, Spe
end end
seek(stream, current_pos) seek(stream, current_pos)
# Create SpectrumAsset objects # CRITICAL FIX: Use correct data types that match the converter output
mz_asset = SpectrumAsset(mz_format, mz_is_compressed, mz_offset, nPoints, :mz) mz_asset = SpectrumAsset(Float64, mz_is_compressed, mz_offset, nPoints, :mz)
int_asset = SpectrumAsset(intensity_format, int_is_compressed, int_offset, nPoints, :intensity) int_asset = SpectrumAsset(Float32, int_is_compressed, int_offset, nPoints, :intensity)
spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset) spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset)
current_ibd_offset += mz_len_bytes + int_len_bytes current_ibd_offset += mz_len_bytes + int_len_bytes
skip(stream, attr[8]) # Skip to the end of the spectrum tag
# Skip to next spectrum
line = readuntil(stream, "</spectrum>")
end end
return spectra_metadata return spectra_metadata
@ -385,8 +432,7 @@ end
function parse_compressed(stream::IO, hIbd::IO, param_groups::Dict{String, SpecDim}, function parse_compressed(stream::IO, hIbd::IO, param_groups::Dict{String, SpecDim},
width::Int32, height::Int32, num_spectra::Int32, width::Int32, height::Int32, num_spectra::Int32,
default_mz_format::DataType, default_intensity_format::DataType, default_mz_format::DataType, default_intensity_format::DataType,
mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode) # Changed Int64 to SpectrumMode mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode)
# New parser for compressed data
spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra) spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra)
# Use concrete type for array_data # Use concrete type for array_data
@ -480,29 +526,35 @@ function parse_compressed(stream::IO, hIbd::IO, param_groups::Dict{String, SpecD
mz_data = filter(d -> d.is_mz, array_data) mz_data = filter(d -> d.is_mz, array_data)
int_data = filter(d -> !d.is_mz, array_data) int_data = filter(d -> !d.is_mz, array_data)
# FIX: Handle empty spectra gracefully instead of throwing errors
current_ibd_pos = position(hIbd)
if length(mz_data) != 1 || length(int_data) != 1 if length(mz_data) != 1 || length(int_data) != 1
error("Spectrum $k: Expected exactly one m/z and one intensity array") # Create placeholder metadata for empty/invalid spectrum
println("DEBUG: Spectrum $k is empty or invalid - creating placeholder metadata")
mz_asset = SpectrumAsset(default_mz_format, mz_is_compressed, current_ibd_pos, 0, :mz)
int_asset = SpectrumAsset(default_intensity_format, int_is_compressed, current_ibd_pos, 0, :intensity)
else
mz_info = mz_data[1]
int_info = int_data[1]
# DEBUG: Print first spectrum details
if k == 1
println("DEBUG First spectrum parsed:")
println(" Coordinates: x=$x, y=$y")
println(" Mode: $spectrum_mode")
println(" m/z array: array_length=$(mz_info.array_length), encoded_length=$(mz_info.encoded_length), offset=$(mz_info.offset)")
println(" intensity array: array_length=$(int_info.array_length), encoded_length=$(int_info.encoded_length), offset=$(int_info.offset)")
println(" Expected m/z bytes: $(mz_info.array_length * sizeof(default_mz_format))")
println(" Expected intensity bytes: $(int_info.array_length * sizeof(default_intensity_format))")
end
# Create SpectrumAsset objects
mz_asset = SpectrumAsset(default_mz_format, mz_is_compressed, mz_info.offset,
mz_is_compressed ? mz_info.encoded_length : mz_info.array_length, :mz)
int_asset = SpectrumAsset(default_intensity_format, int_is_compressed, int_info.offset,
int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity)
end end
mz_info = mz_data[1]
int_info = int_data[1]
# DEBUG: Print first spectrum details
if k == 1
println("DEBUG First spectrum parsed:")
println(" Coordinates: x=$x, y=$y")
println(" Mode: $spectrum_mode")
println(" m/z array: array_length=$(mz_info.array_length), encoded_length=$(mz_info.encoded_length), offset=$(mz_info.offset)")
println(" intensity array: array_length=$(int_info.array_length), encoded_length=$(int_info.encoded_length), offset=$(int_info.offset)")
println(" Expected m/z bytes: $(mz_info.array_length * sizeof(default_mz_format))")
println(" Expected intensity bytes: $(int_info.array_length * sizeof(default_intensity_format))")
end
# Create SpectrumAsset objects
mz_asset = SpectrumAsset(default_mz_format, mz_is_compressed, mz_info.offset,
mz_is_compressed ? mz_info.encoded_length : mz_info.array_length, :mz)
int_asset = SpectrumAsset(default_intensity_format, int_is_compressed, int_info.offset,
int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity)
spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset) spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset)
end end
@ -514,7 +566,6 @@ function parse_neofx(stream::IO, hIbd::IO, param_groups::Dict{String, SpecDim},
width::Int32, height::Int32, num_spectra::Int32, width::Int32, height::Int32, num_spectra::Int32,
default_mz_format::DataType, default_intensity_format::DataType, default_mz_format::DataType, default_intensity_format::DataType,
mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode) mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode)
# New parser for compressed data
spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra) spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra)
array_data_type = @NamedTuple{is_mz::Bool, array_length::Int32, encoded_length::Int64, offset::Int64} array_data_type = @NamedTuple{is_mz::Bool, array_length::Int32, encoded_length::Int64, offset::Int64}
@ -607,29 +658,35 @@ function parse_neofx(stream::IO, hIbd::IO, param_groups::Dict{String, SpecDim},
mz_data = filter(d -> d.is_mz, array_data) mz_data = filter(d -> d.is_mz, array_data)
int_data = filter(d -> !d.is_mz, array_data) int_data = filter(d -> !d.is_mz, array_data)
# FIX: Handle empty spectra gracefully instead of throwing errors
current_ibd_pos = position(hIbd)
if length(mz_data) != 1 || length(int_data) != 1 if length(mz_data) != 1 || length(int_data) != 1
error("Spectrum $k: Expected exactly one m/z and one intensity array") # Create placeholder metadata for empty/invalid spectrum
println("DEBUG: Spectrum $k is empty or invalid - creating placeholder metadata")
mz_asset = SpectrumAsset(default_mz_format, mz_is_compressed, current_ibd_pos, 0, :mz)
int_asset = SpectrumAsset(default_intensity_format, int_is_compressed, current_ibd_pos, 0, :intensity)
else
mz_info = mz_data[1]
int_info = int_data[1]
# DEBUG: Print first spectrum details
if k == 1
println("DEBUG First spectrum parsed:")
println(" Coordinates: x=$x, y=$y")
println(" Mode: $spectrum_mode")
println(" m/z array: array_length=$(mz_info.array_length), encoded_length=$(mz_info.encoded_length), offset=$(mz_info.offset)")
println(" intensity array: array_length=$(int_info.array_length), encoded_length=$(int_info.encoded_length), offset=$(int_info.offset)")
println(" Expected m/z bytes: $(mz_info.array_length * sizeof(default_mz_format))")
println(" Expected intensity bytes: $(int_info.array_length * sizeof(default_intensity_format))")
end
# Create SpectrumAsset objects
mz_asset = SpectrumAsset(default_mz_format, mz_is_compressed, mz_info.offset,
mz_is_compressed ? mz_info.encoded_length : mz_info.array_length, :mz)
int_asset = SpectrumAsset(default_intensity_format, int_is_compressed, int_info.offset,
int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity)
end end
mz_info = mz_data[1]
int_info = int_data[1]
# DEBUG: Print first spectrum details
if k == 1
println("DEBUG First spectrum parsed:")
println(" Coordinates: x=$x, y=$y")
println(" Mode: $spectrum_mode")
println(" m/z array: array_length=$(mz_info.array_length), encoded_length=$(mz_info.encoded_length), offset=$(mz_info.offset)")
println(" intensity array: array_length=$(int_info.array_length), encoded_length=$(int_info.encoded_length), offset=$(int_info.offset)")
println(" Expected m/z bytes: $(mz_info.array_length * sizeof(default_mz_format))")
println(" Expected intensity bytes: $(int_info.array_length * sizeof(default_intensity_format))")
end
# Create SpectrumAsset objects
mz_asset = SpectrumAsset(default_mz_format, mz_is_compressed, mz_info.offset,
mz_is_compressed ? mz_info.encoded_length : mz_info.array_length, :mz)
int_asset = SpectrumAsset(default_intensity_format, int_is_compressed, int_info.offset,
int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity)
spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset) spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset)
end end
@ -656,6 +713,11 @@ This optimized version uses binary search for efficiency.
""" """
function find_mass(mz_array::AbstractVector{<:Real}, intensity_array::AbstractVector{<:Real}, function find_mass(mz_array::AbstractVector{<:Real}, intensity_array::AbstractVector{<:Real},
target_mass::Real, tolerance::Real) target_mass::Real, tolerance::Real)
# Fast-path rejection: if the array is empty or the target is out of range
if isempty(mz_array) || target_mass + tolerance < first(mz_array) || target_mass - tolerance > last(mz_array)
return 0.0
end
lower_bound = target_mass - tolerance lower_bound = target_mass - tolerance
upper_bound = target_mass + tolerance upper_bound = target_mass + tolerance
@ -811,9 +873,12 @@ This is a highly performant function that iterates through the full dataset only
function get_multiple_mz_slices(data::MSIData, masses::AbstractVector{<:Real}, tolerance::Real; mask_path::Union{String, Nothing}=nothing) function get_multiple_mz_slices(data::MSIData, masses::AbstractVector{<:Real}, tolerance::Real; mask_path::Union{String, Nothing}=nothing)
width, height = data.image_dims width, height = data.image_dims
# Sort masses to improve cache locality during search
sorted_masses = sort(masses)
# 1. Initialize a dictionary to hold the output slice matrices # 1. Initialize a dictionary to hold the output slice matrices
slice_dict = Dict{Real, Matrix{Float64}}() slice_dict = Dict{Real, Matrix{Float64}}()
for mass in masses for mass in sorted_masses
slice_dict[mass] = zeros(Float64, height, width) slice_dict[mass] = zeros(Float64, height, width)
end end
@ -836,7 +901,7 @@ function get_multiple_mz_slices(data::MSIData, masses::AbstractVector{<:Real}, t
indices_to_check = masked_indices === nothing ? (1:length(data.spectra_metadata)) : masked_indices indices_to_check = masked_indices === nothing ? (1:length(data.spectra_metadata)) : masked_indices
# 3. Find all spectra that could contain *any* of the requested masses. # 3. Find all spectra that could contain *any* of the requested masses.
for mass in masses for mass in sorted_masses
target_min = mass - tolerance target_min = mass - tolerance
target_max = mass + tolerance target_max = mass + tolerance
for i in indices_to_check for i in indices_to_check
@ -858,7 +923,7 @@ function get_multiple_mz_slices(data::MSIData, masses::AbstractVector{<:Real}, t
_iterate_spectra_fast(data, collect(candidate_indices)) do idx, mz_array, intensity_array _iterate_spectra_fast(data, collect(candidate_indices)) do idx, mz_array, intensity_array
meta = data.spectra_metadata[idx] meta = data.spectra_metadata[idx]
# For this single spectrum, check all masses of interest # For this single spectrum, check all masses of interest
for mass in masses for mass in sorted_masses
# Check if this spectrum's range actually covers the current mass # Check if this spectrum's range actually covers the current mass
# This is a finer-grained check than the initial filtering # This is a finer-grained check than the initial filtering
if !isempty(mz_array) && (mass + tolerance) >= first(mz_array) && (mass - tolerance) <= last(mz_array) if !isempty(mz_array) && (mass + tolerance) >= first(mz_array) && (mass - tolerance) <= last(mz_array)
@ -873,7 +938,7 @@ function get_multiple_mz_slices(data::MSIData, masses::AbstractVector{<:Real}, t
end end
# 5. Clean up and return # 5. Clean up and return
for mass in masses for mass in sorted_masses
replace!(slice_dict[mass], NaN => 0.0) replace!(slice_dict[mass], NaN => 0.0)
end end
@ -1066,24 +1131,21 @@ within these bounds.
function TrIQ(pixMap::AbstractMatrix{<:Real}, depth::Integer, prob::Real=0.98; mask_matrix::Union{BitMatrix, Nothing}=nothing) function TrIQ(pixMap::AbstractMatrix{<:Real}, depth::Integer, prob::Real=0.98; mask_matrix::Union{BitMatrix, Nothing}=nothing)
local values_for_thres local values_for_thres
if mask_matrix !== nothing if mask_matrix !== nothing
# Extract values only from the masked region for threshold calculation
values_for_thres = pixMap[mask_matrix] values_for_thres = pixMap[mask_matrix]
# Filter out only zeros created by the mask.
filter!(x -> x != 0, values_for_thres) filter!(x -> x != 0, values_for_thres)
else else
values_for_thres = pixMap values_for_thres = pixMap
end end
# If the relevant values are empty or all zero, return a zero matrix
if isempty(values_for_thres) || all(iszero, values_for_thres) if isempty(values_for_thres) || all(iszero, values_for_thres)
bounds = (0.0, 0.0) bounds = (0.0, 0.0)
quantized = zeros(UInt8, size(pixMap))
else else
# Compute new dynamic range from the (potentially filtered) values
bounds = get_outlier_thres(values_for_thres, prob) bounds = get_outlier_thres(values_for_thres, prob)
quantized = set_pixel_depth(pixMap, bounds, depth)
end end
# Set intensity dynamic range for the *entire* pixMap, using bounds derived from masked data return quantized, bounds # Return both!
return set_pixel_depth(pixMap, bounds, depth)
end end
""" """
@ -1134,22 +1196,18 @@ function quantize_intensity(slice::AbstractMatrix{<:Real}, levels::Integer=256;
end end
if max_val <= 0 if max_val <= 0
return zeros(UInt8, size(slice)) bounds = (0.0, 0.0)
quantized = zeros(UInt8, size(slice))
else
bounds = (0.0, max_val)
quantized = similar(slice, UInt8)
scale = (levels - 1) / max_val
@inbounds for i in eachindex(slice)
quantized[i] = round(UInt8, clamp(slice[i] * scale, 0, levels - 1))
end
end end
# Scale relative to the absolute maximum value to preserve the zero point. return quantized, bounds # Return both!
# The original logic used 'colorLevel' which was the max value (e.g., 255).
scale = (levels - 1) / max_val
# Pre-allocate result for better performance
result = similar(slice, UInt8)
# Use inbounds for faster access
@inbounds for i in eachindex(slice)
result[i] = round(UInt8, clamp(slice[i] * scale, 0, levels - 1))
end
return result
end end
""" """
@ -1523,28 +1581,25 @@ end
# Viridis color palette (256 colors) - defined as constant # Viridis color palette (256 colors) - defined as constant
const ViridisPalette = generate_palette(ColorSchemes.viridis) const ViridisPalette = generate_palette(ColorSchemes.viridis)
function generate_colorbar_image(slice_data::AbstractMatrix, color_levels::Int, output_path::String; use_triq::Bool=false, triq_prob::Float64=0.98, mask_path::Union{String, Nothing}=nothing) function generate_colorbar_image(slice_data::AbstractMatrix, color_levels::Int, output_path::String,
# 1. Determine bounds based on whether TrIQ is used and if a mask is applied bounds::Tuple{Float64, Float64};
local data_for_bounds use_triq::Bool=false, triq_prob::Float64=0.98,
if mask_path !== nothing mask_path::Union{String, Nothing}=nothing)
height, width = size(slice_data) # Use the provided bounds instead of recalculating
mask_matrix = load_and_prepare_mask(mask_path, (width, height)) min_val, max_val = bounds
# Extract only the non-zero values within the mask to accurately calculate the range
data_for_bounds = slice_data[mask_matrix] # If bounds are invalid, calculate fallback
filter!(x -> x > 0, data_for_bounds) if min_val == max_val == 0.0
else local data_for_bounds
data_for_bounds = vec(slice_data) if mask_path !== nothing
end height, width = size(slice_data)
mask_matrix = load_and_prepare_mask(mask_path, (width, height))
# Handle cases where data_for_bounds might be empty after filtering data_for_bounds = slice_data[mask_matrix]
if isempty(data_for_bounds) filter!(x -> x > 0, data_for_bounds)
min_val, max_val = 0.0, 1.0 # Default range if no data in ROI
else
min_val, max_val = if use_triq
get_outlier_thres(data_for_bounds, triq_prob)
else else
extrema(data_for_bounds) data_for_bounds = vec(slice_data)
end end
min_val, max_val = isempty(data_for_bounds) ? (0.0, 1.0) : extrema(data_for_bounds)
end end
# 2. Replicate the tick calculation logic from plot_slices # 2. Replicate the tick calculation logic from plot_slices

17
src/mzML.jl
View File

@ -36,10 +36,13 @@ determine the data type, compression, and axis type.
function get_spectrum_asset_metadata(stream::IO) function get_spectrum_asset_metadata(stream::IO)
start_pos = position(stream) start_pos = position(stream)
bda_tag = find_tag(stream, r"<binaryDataArray\s+encodedLength=\"(\d+)\"" ) bda_tag = find_tag(stream, r"<binaryDataArray\s+encodedLength=\"(\d+)\"")
if bda_tag === nothing
error("Cannot find binaryDataArray") if bda_tag === nothing
end
throw(FileFormatError("Cannot find binaryDataArray"))
end
encoded_length = parse(Int32, bda_tag.captures[1]) encoded_length = parse(Int32, bda_tag.captures[1])
# Initialize parameters as separate variables with concrete types # Initialize parameters as separate variables with concrete types
@ -202,7 +205,7 @@ function find_index_offset(stream::IO)::Int64
index_offset_match = match(r"<indexListOffset>(\d+)</indexListOffset>", footer) index_offset_match = match(r"<indexListOffset>(\d+)</indexListOffset>", footer)
if index_offset_match === nothing if index_offset_match === nothing
error("Could not find <indexListOffset>. File may not be an indexed mzML.") throw(FileFormatError("Could not find <indexListOffset>. File may not be an indexed mzML."))
end end
return parse(Int64, index_offset_match.captures[1]) return parse(Int64, index_offset_match.captures[1])
@ -235,14 +238,14 @@ function load_mzml_lazy(file_path::String; cache_size::Int=100)
println("DEBUG: Searching for '<index name=\"spectrum\">'.") println("DEBUG: Searching for '<index name=\"spectrum\">'.")
if find_tag(stream, r"<index\s+name=\"spectrum\"") === nothing if find_tag(stream, r"<index\s+name=\"spectrum\"") === nothing
error("Could not find spectrum index.") throw(FileFormatError("Could not find spectrum index."))
end end
println("DEBUG: Found spectrum index tag.") println("DEBUG: Found spectrum index tag.")
println("DEBUG: Parsing spectrum offsets...") println("DEBUG: Parsing spectrum offsets...")
spectrum_offsets = parse_offset_list(stream) spectrum_offsets = parse_offset_list(stream)
if isempty(spectrum_offsets) if isempty(spectrum_offsets)
error("No spectrum offsets found.") throw(FileFormatError("No spectrum offsets found."))
end end
num_spectra = length(spectrum_offsets) num_spectra = length(spectrum_offsets)
println("DEBUG: Found $num_spectra spectrum offsets.") println("DEBUG: Found $num_spectra spectrum offsets.")

3
start_MSI_GUI.jl
View File

@ -13,6 +13,9 @@ using Genie
# Load and configure Genie # Load and configure Genie
Genie.loadapp() Genie.loadapp()
# Remove html parser error discrepancy
redirect_stderr(devnull)
# Start the Genie server # Start the Genie server
@async begin @async begin
up(host="127.0.0.1", port=1481) up(host="127.0.0.1", port=1481)

4
test/run_tests.jl
View File

@ -76,8 +76,8 @@ const COORDS_TO_PLOT = (50, 50) # Example coordinates (X, Y)
const RESULTS_DIR = "test/results" const RESULTS_DIR = "test/results"
test1 = false test1 = false
test2 = false test2 = true
test3 = true test3 = false
# =================================================================== # ===================================================================
# DATA VALIDATION UTILITY # DATA VALIDATION UTILITY