UI now has support for mzml to imzml conversion, made small optimizations to the src folder's API to improve performance
This commit is contained in:
parent
2fd2b90513
commit
39fe2bf52c
5
.gitignore
vendored
5
.gitignore
vendored
@ -1,8 +1,9 @@
|
||||
public/*
|
||||
public/css/imgOver.png
|
||||
!public/css/
|
||||
!public/css/autogenerated.css
|
||||
!public/css/LABI_logo.png
|
||||
!public/masks/
|
||||
public/masks/*
|
||||
!public/masks/static.txt
|
||||
log/*
|
||||
R_original_scripts/
|
||||
test/results/
|
||||
|
||||
76
app.jl
76
app.jl
@ -18,7 +18,7 @@ using Base.Filesystem: mv # To rename files in the system
|
||||
using Printf # Required for @sprintf macro in colorbar generation
|
||||
|
||||
# Bring MSIData into App module's scope
|
||||
using .MSI_src: MSIData, OpenMSIData, GetSpectrum, IterateSpectra, ImzMLSource, _iterate_spectra_fast, MzMLSource, find_mass, ViridisPalette, get_mz_slice, quantize_intensity, save_bitmap, median_filter, save_bitmap, downsample_spectrum, TrIQ, precompute_analytics
|
||||
using .MSI_src: MSIData, OpenMSIData, #=GetSpectrum,=# process_spectrum, IterateSpectra, ImzMLSource, _iterate_spectra_fast, MzMLSource, find_mass, ViridisPalette, get_mz_slice, quantize_intensity, save_bitmap, median_filter, save_bitmap, downsample_spectrum, TrIQ, precompute_analytics, ImportMzmlFile
|
||||
|
||||
include("./julia_imzML_visual.jl")
|
||||
|
||||
@ -130,6 +130,17 @@ include("./julia_imzML_visual.jl")
|
||||
# Saves the route where imzML and mzML files are located
|
||||
@out full_route=""
|
||||
|
||||
# == Converter Tab Variables ==
|
||||
@in left_tab = "generator"
|
||||
@out mzml_full_route = ""
|
||||
@out sync_full_route = ""
|
||||
@in btnSearchMzml = false
|
||||
@in btnSearchSync = false
|
||||
@in convert_process = false
|
||||
@out progress_conversion = false
|
||||
@out msg_conversion = ""
|
||||
@out btnConvertDisable = true
|
||||
|
||||
|
||||
# For the creation of images with a more specific mass charge
|
||||
@out text_nmass=""
|
||||
@ -394,6 +405,69 @@ include("./julia_imzML_visual.jl")
|
||||
showMetadataDialog = true
|
||||
end
|
||||
|
||||
@onchange btnSearchMzml, btnSearchSync begin
|
||||
if btnSearchMzml
|
||||
picked_route = pick_file(; filterlist="mzML,mzml")
|
||||
if !isempty(picked_route)
|
||||
mzml_full_route = picked_route
|
||||
end
|
||||
btnSearchMzml = false # Reset the button
|
||||
end
|
||||
|
||||
if btnSearchSync
|
||||
picked_route = pick_file(; filterlist="txt")
|
||||
if !isempty(picked_route)
|
||||
sync_full_route = picked_route
|
||||
end
|
||||
btnSearchSync = false # Reset the button
|
||||
end
|
||||
|
||||
# Enable button only if both files are selected
|
||||
btnConvertDisable = isempty(mzml_full_route) || isempty(sync_full_route)
|
||||
end
|
||||
|
||||
@onbutton convert_process begin
|
||||
if isempty(mzml_full_route) || isempty(sync_full_route)
|
||||
msg_conversion = "Please select both an .mzML file and a .txt sync file."
|
||||
warning_msg = true
|
||||
return
|
||||
end
|
||||
|
||||
progress_conversion = true
|
||||
btnConvertDisable = true
|
||||
msg_conversion = "Starting conversion process..."
|
||||
|
||||
@async begin
|
||||
try
|
||||
sTime = time()
|
||||
target_imzml = replace(mzml_full_route, r"\.(mzml|mzML)$" => ".imzML")
|
||||
|
||||
msg_conversion = "Converting $(basename(mzml_full_route)) to $(basename(target_imzml))... This may take a while."
|
||||
|
||||
success = ImportMzmlFile(mzml_full_route, sync_full_route, target_imzml)
|
||||
|
||||
fTime = time()
|
||||
eTime = round(fTime - sTime, digits=3)
|
||||
|
||||
if success
|
||||
msg_conversion = "Conversion successful in $(eTime) seconds. Output file: $(basename(target_imzml))"
|
||||
else
|
||||
msg_conversion = "Conversion failed after $(eTime) seconds. Check console for errors."
|
||||
warning_msg = true
|
||||
end
|
||||
|
||||
catch e
|
||||
msg_conversion = "An error occurred during conversion: $e"
|
||||
warning_msg = true
|
||||
@error "Conversion failed" exception=(e, catch_backtrace())
|
||||
finally
|
||||
progress_conversion = false
|
||||
# Re-enable button if files are still selected
|
||||
btnConvertDisable = isempty(mzml_full_route) || isempty(sync_full_route)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@onbutton mainProcess @time begin
|
||||
# UI updates immediately
|
||||
progress = true
|
||||
|
||||
44
app.jl.html
44
app.jl.html
@ -2,17 +2,23 @@
|
||||
<img src="/css/LABI_logo.png" alt="Labi Logo Icon" id="imgLogo">
|
||||
<div>
|
||||
<h4>JuliaMSI </h4>
|
||||
<h6>Please make sure the ibd, the mzML and the imzML files are located in the same directory and have the same
|
||||
name.</h6>
|
||||
<h6>It may take a while to generate the image and the plot, please be patient.</h6>
|
||||
<h6>To generate the contour or surface plots, you have to select the desired image first using the interface.</h6>
|
||||
</div>
|
||||
</header>
|
||||
<div id="extDivStyle" class="row col-12 q-pa-xl">
|
||||
<div class="row col-6">
|
||||
<!-- Left DIV -->
|
||||
<div id="intDivStyle-left" class="st-col col-12 st-module">
|
||||
<h6>Search for the imzML or mzML file in your system</h6>
|
||||
<q-tabs v-model="left_tab" dense class="text-grey" indicator-color="primary" align="justify">
|
||||
<q-tab name="generator" label="Slice Generator"></q-tab>
|
||||
<q-tab name="converter" label="Converter"></q-tab>
|
||||
</q-tabs>
|
||||
<q-separator></q-separator>
|
||||
<q-tab-panels v-model="left_tab" animated>
|
||||
<q-tab-panel name="generator">
|
||||
<div class="text-h6">imzML & mzML Data Processor</div>
|
||||
<p>Please make sure the ibd and imzML file are located in the same directory and have the same name.
|
||||
<br>It may take a while to generate the image and the plot, please be patient.
|
||||
<br>To generate the contour or surface plots, you have to select the desired image first using the interface.</p>
|
||||
<q-input standout="custom-standout" class="q-ma-sm cursor-pointer" v-model="full_route" readonly label="Select your imzML or mzML file" v-on:click="btnSearch=true">
|
||||
<template v-slot:append>
|
||||
<q-icon name="search" v:onclick="btnSearch=true" class="cursor-pointer" />
|
||||
@ -99,7 +105,7 @@
|
||||
</div>
|
||||
<div class="row">
|
||||
<q-btn :loading="progress" class="q-ma-sm btn-style" :disabled="btnStartDisable" icon="play_arrow"
|
||||
v-on:click="mainProcess=true" padding="lg" label="Generate Spectra">
|
||||
v-on:click="mainProcess=true" padding="lg" label="Generate Slice">
|
||||
<template v-slot:loading>
|
||||
<q-spinner-hourglass class="on-left" />
|
||||
Loading...
|
||||
@ -164,6 +170,32 @@
|
||||
<q-badge style="background-color: #009f90;"> Transparency: {{ imgTrans }}</q-badge>
|
||||
</div>
|
||||
</div>
|
||||
</q-tab-panel>
|
||||
<q-tab-panel name="converter">
|
||||
<div class="text-h6">mzML to imzML Converter</div>
|
||||
<p>Select the .mzML file and the corresponding .txt synchronization file to convert them into an .imzML/.ibd pair.</p>
|
||||
|
||||
<q-input standout="custom-standout" class="q-ma-sm cursor-pointer" v-model="mzml_full_route" readonly label="Select your .mzML file" v-on:click="btnSearchMzml=true">
|
||||
<template v-slot:append>
|
||||
<q-icon name="search" v:onclick="btnSearchMzml=true" class="cursor-pointer" />
|
||||
</template>
|
||||
</q-input>
|
||||
|
||||
<q-input standout="custom-standout" class="q-ma-sm cursor-pointer" v-model="sync_full_route" readonly label="Select your .txt sync file" v-on:click="btnSearchSync=true">
|
||||
<template v-slot:append>
|
||||
<q-icon name="search" v:onclick="btnSearchSync=true" class="cursor-pointer" />
|
||||
</template>
|
||||
</q-input>
|
||||
|
||||
<q-btn :loading="progress_conversion" class="q-ma-sm btn-style" :disabled="btnConvertDisable" icon="swap_horiz" v-on:click="convert_process=true" padding="lg" label="Convert File">
|
||||
<template v-slot:loading>
|
||||
<q-spinner-hourglass class="on-left" />
|
||||
Converting...
|
||||
</template>
|
||||
</q-btn>
|
||||
<p>{{msg_conversion}}</p>
|
||||
</q-tab-panel>
|
||||
</q-tab-panels>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row col-6">
|
||||
|
||||
@ -502,13 +502,21 @@ function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int,
|
||||
x = clamp(xCoord, 1, imgWidth)
|
||||
y = clamp(yCoord, 1, imgHeight)
|
||||
|
||||
mz, intensity = GetSpectrum(data, Int(x), Int(y))
|
||||
# mz, intensity = GetSpectrum(data, Int(x), Int(y))
|
||||
process_spectrum(data, Int(x), Int(y)) do recieved_mz, recieved_intensity
|
||||
mz = recieved_mz
|
||||
intensity = recieved_intensity
|
||||
end
|
||||
plot_title = "Spectrum at ($x, $y)"
|
||||
else
|
||||
# For non-imaging data, treat xCoord as the spectrum index
|
||||
index = clamp(xCoord, 1, length(data.spectra_metadata))
|
||||
|
||||
mz, intensity = GetSpectrum(data, index)
|
||||
# mz, intensity = GetSpectrum(data, index)
|
||||
process_spectrum(data, index) do recieved_mz, recieved_intensity
|
||||
mz = recieved_mz
|
||||
intensity = recieved_intensity
|
||||
end
|
||||
plot_title = "Spectrum #$index"
|
||||
end
|
||||
|
||||
|
||||
@ -45,6 +45,51 @@
|
||||
color: #009f90;
|
||||
}
|
||||
|
||||
#intDivStyle-left {
|
||||
border-radius: 10px;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
/* Tab styling to match your theme */
|
||||
.q-tabs {
|
||||
background-color: transparent !important;
|
||||
}
|
||||
|
||||
.q-tab {
|
||||
color: #7f8389 !important;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.q-tab--active {
|
||||
color: #009f90 !important;
|
||||
background-color: rgba(0, 159, 144, 0.1) !important;
|
||||
border-radius: 5px;
|
||||
}
|
||||
|
||||
.q-tabs[active-color="primary"] .q-tab--active {
|
||||
color: #009f90 !important;
|
||||
}
|
||||
|
||||
/* Indicator color to match your primary color */
|
||||
.q-tab__indicator {
|
||||
background-color: #009f90 !important;
|
||||
}
|
||||
|
||||
/* Dense tabs adjustment */
|
||||
.q-tabs--dense .q-tab {
|
||||
padding: 8px 12px;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
#intDivStyle-left .q-tab-panels {
|
||||
height: 670px; /* Set this to accommodate your tallest content */
|
||||
}
|
||||
|
||||
#intDivStyle-left .q-tab-panel {
|
||||
height: 100%;
|
||||
overflow-y: auto; /* Add scroll if content overflows */
|
||||
}
|
||||
|
||||
*{
|
||||
font-family: 'Roboto', 'Lato', sans-serif;
|
||||
}
|
||||
1
public/css/themes/static.txt
Normal file
1
public/css/themes/static.txt
Normal file
@ -0,0 +1 @@
|
||||
themes
|
||||
1
public/masks/static.txt
Normal file
1
public/masks/static.txt
Normal file
@ -0,0 +1 @@
|
||||
masks
|
||||
@ -328,6 +328,42 @@ function GetSpectrum(data::MSIData, x::Int, y::Int)
|
||||
return GetSpectrum(data, index) # Call the existing method
|
||||
end
|
||||
|
||||
"""
|
||||
process_spectrum(f::Function, data::MSIData, index::Int)
|
||||
|
||||
A "function barrier" helper for safely processing a single spectrum.
|
||||
|
||||
This function retrieves a spectrum and then immediately calls the provided function `f`
|
||||
with the resulting `(mz, intensity)` arrays. This pattern is crucial for performance.
|
||||
While `GetSpectrum` itself is type-unstable (because the array data types are not
|
||||
known at compile time), calling `f` with the result allows Julia's compiler to
|
||||
generate specialized, fast code for `f` based on the *concrete* types it receives.
|
||||
|
||||
Use this function when you need to perform performance-critical operations on a
|
||||
single spectrum. Your logic should be inside the function passed to this helper.
|
||||
```
|
||||
"""
|
||||
function process_spectrum(f::Function, data::MSIData, index::Int)
|
||||
# This call is type-unstable.
|
||||
mz, intensity = GetSpectrum(data, index)
|
||||
|
||||
# By immediately calling `f`, we create a "function barrier".
|
||||
# Julia will compile a specialized version of `f` for the
|
||||
# concrete types of `mz` and `intensity`.
|
||||
return f(mz, intensity)
|
||||
end
|
||||
|
||||
"""
|
||||
process_spectrum(f::Function, data::MSIData, x::Int, y::Int)
|
||||
|
||||
A coordinate-based version of the "function barrier" helper. See `process_spectrum`
|
||||
for details on the performance pattern.
|
||||
"""
|
||||
function process_spectrum(f::Function, data::MSIData, x::Int, y::Int)
|
||||
mz, intensity = GetSpectrum(data, x, y)
|
||||
return f(mz, intensity)
|
||||
end
|
||||
|
||||
"""
|
||||
precompute_analytics(msi_data::MSIData)
|
||||
|
||||
@ -680,15 +716,11 @@ This is effectively the total spectrum divided by the number of spectra.
|
||||
Returns a tuple containing two vectors: the binned m/z axis and the averaged intensities.
|
||||
"""
|
||||
function get_average_spectrum(msi_data::MSIData; num_bins::Int=2000)
|
||||
# This function uses the exact same logic as get_total_spectrum...
|
||||
mz_bins, intensity_sum = get_total_spectrum(msi_data, num_bins=num_bins)
|
||||
|
||||
if isempty(intensity_sum)
|
||||
return (mz_bins, intensity_sum)
|
||||
end
|
||||
|
||||
# ...with one final step: dividing by the number of spectra to get the average.
|
||||
println("Averaging spectrum...")
|
||||
num_spectra = length(msi_data.spectra_metadata)
|
||||
average_intensity = intensity_sum ./ num_spectra
|
||||
|
||||
|
||||
@ -107,19 +107,12 @@ function GetMzmlScanTime_linebyline(fileName::String)
|
||||
end
|
||||
end
|
||||
|
||||
sort!(times, by = x -> x[1])
|
||||
|
||||
if !isempty(times)
|
||||
first_time = times[1][2]
|
||||
for i in eachindex(times)
|
||||
times[i] = (times[i][1], times[i][2] - first_time)
|
||||
result = Matrix{Int64}(undef, length(times), 2)
|
||||
for (i, t) in enumerate(times)
|
||||
result[i, 1] = t[1]
|
||||
result[i, 2] = t[2]
|
||||
end
|
||||
end
|
||||
|
||||
if isempty(times)
|
||||
return Matrix{Int64}(undef, 0, 2)
|
||||
end
|
||||
return permutedims(hcat(collect.(times)...))
|
||||
return result
|
||||
end
|
||||
|
||||
"""
|
||||
@ -209,10 +202,12 @@ function GetMzmlScanTime(fileName::String)
|
||||
end
|
||||
end
|
||||
|
||||
if isempty(times)
|
||||
return Matrix{Int64}(undef, 0, 2)
|
||||
result = Matrix{Int64}(undef, length(times), 2)
|
||||
for (i, t) in enumerate(times)
|
||||
result[i, 1] = t[1]
|
||||
result[i, 2] = t[2]
|
||||
end
|
||||
return permutedims(hcat(collect.(times)...))
|
||||
return result
|
||||
end
|
||||
|
||||
"""
|
||||
@ -399,14 +394,27 @@ This function handles three cases:
|
||||
# Returns
|
||||
- A tuple `(mz_array, intensity_array)` for the rendered pixel spectrum.
|
||||
"""
|
||||
function RenderPixel(pixel_info, scans, msi_data::MSIData, scan_time_deltas, pixel_time_deltas)
|
||||
function RenderPixel(
|
||||
pixel_info::AbstractVector{Int64},
|
||||
scans::AbstractMatrix{Int64},
|
||||
msi_data::MSIData,
|
||||
scan_time_deltas::AbstractVector{Int64},
|
||||
pixel_time_deltas::AbstractVector{Int64}
|
||||
)
|
||||
pixel_time = pixel_info[3]
|
||||
first_scan = pixel_info[4]
|
||||
last_scan = pixel_info[5]
|
||||
num_actions = last_scan - first_scan
|
||||
|
||||
# Get reference m/z array from the first scan involved
|
||||
# Get reference m/z array from the first scan involved.
|
||||
# This call remains type-unstable, but its impact is now isolated and only paid once.
|
||||
mz_array, _ = GetSpectrum(msi_data, first_scan)
|
||||
|
||||
# If the first spectrum was empty, we can't do anything else.
|
||||
if isempty(mz_array)
|
||||
return (mz_array, Float32[])
|
||||
end
|
||||
|
||||
new_intensity = zeros(Float32, length(mz_array))
|
||||
|
||||
# SAFETY: Ensure we have valid scan indices
|
||||
@ -415,41 +423,47 @@ function RenderPixel(pixel_info, scans, msi_data::MSIData, scan_time_deltas, pix
|
||||
end
|
||||
|
||||
if num_actions == 0 # Single scan contributes to the pixel
|
||||
_, intensity = GetSpectrum(msi_data, first_scan)
|
||||
process_spectrum(msi_data, first_scan) do _, intensity
|
||||
# SAFETY: Ensure positive scaling
|
||||
scale = max(pixel_time_deltas[first_scan] / scan_time_deltas[first_scan], 0.0f0)
|
||||
new_intensity .= intensity .* scale
|
||||
end
|
||||
|
||||
elseif num_actions == 1 # Two partial scans contribute
|
||||
# First partial scan
|
||||
_, intensity1 = GetSpectrum(msi_data, first_scan)
|
||||
process_spectrum(msi_data, first_scan) do _, intensity1
|
||||
scale1 = max((scans[last_scan, 2] - pixel_time) / scan_time_deltas[first_scan], 0.0f0)
|
||||
new_intensity .+= intensity1 .* scale1
|
||||
end
|
||||
|
||||
# Second partial scan
|
||||
_, intensity2 = GetSpectrum(msi_data, last_scan)
|
||||
process_spectrum(msi_data, last_scan) do _, intensity2
|
||||
next_pixel_time = pixel_time + pixel_time_deltas[first_scan]
|
||||
scale2 = max((next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan], 0.0f0)
|
||||
new_intensity .+= intensity2 .* scale2
|
||||
end
|
||||
|
||||
elseif num_actions > 1 # Multiple scans contribute
|
||||
# First partial scan
|
||||
_, intensity1 = GetSpectrum(msi_data, first_scan)
|
||||
process_spectrum(msi_data, first_scan) do _, intensity1
|
||||
scale1 = max((scans[first_scan + 1, 2] - pixel_time) / scan_time_deltas[first_scan], 0.0f0)
|
||||
new_intensity .+= intensity1 .* scale1
|
||||
end
|
||||
|
||||
# Full scans in the middle
|
||||
for i in (first_scan + 1):(last_scan - 1)
|
||||
_, intensity_middle = GetSpectrum(msi_data, i)
|
||||
process_spectrum(msi_data, i) do _, intensity_middle
|
||||
new_intensity .+= intensity_middle
|
||||
end
|
||||
end
|
||||
|
||||
# Last partial scan
|
||||
_, intensity2 = GetSpectrum(msi_data, last_scan)
|
||||
process_spectrum(msi_data, last_scan) do _, intensity2
|
||||
next_pixel_time = pixel_time + pixel_time_deltas[first_scan]
|
||||
scale2 = max((next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan], 0.0f0)
|
||||
new_intensity .+= intensity2 .* scale2
|
||||
end
|
||||
end
|
||||
|
||||
# FINAL SAFETY: Clamp any negative values to zero
|
||||
new_intensity = max.(new_intensity, 0.0f0)
|
||||
|
||||
185
src/imzML.jl
185
src/imzML.jl
@ -25,7 +25,7 @@ Core Functions:
|
||||
|
||||
Determines the storage order of the m/z and intensity arrays.
|
||||
"""
|
||||
function axes_config_img(stream)
|
||||
function axes_config_img(stream::IO)
|
||||
param_groups = Dict{String, SpecDim}()
|
||||
find_tag(stream, r"<referenceableParamGroupList")
|
||||
|
||||
@ -52,10 +52,10 @@ end
|
||||
|
||||
Reads the maximum X and Y dimensions and total spectrum count from the metadata.
|
||||
"""
|
||||
function get_img_dimensions(stream)
|
||||
function get_img_dimensions(stream::IO)
|
||||
find_tag(stream, r"^\s*<(scanSettings )")
|
||||
n = 2
|
||||
dim = [0, 0, 0]
|
||||
dim = zeros(Int32, 3)
|
||||
|
||||
while n > 0 && !eof(stream)
|
||||
currLine = readline(stream)
|
||||
@ -86,7 +86,7 @@ end
|
||||
|
||||
Calculates the character offset within a `<spectrum>` tag, ignoring attribute values.
|
||||
"""
|
||||
function get_spectrum_tag_offset(stream)
|
||||
function get_spectrum_tag_offset(stream::IO)
|
||||
offset = position(stream)
|
||||
tag = find_tag(stream, r"^\s*<spectrum (.+)")
|
||||
first = 1
|
||||
@ -107,7 +107,7 @@ end
|
||||
|
||||
Reads metadata to determine the byte offsets and data types for reading spectra.
|
||||
"""
|
||||
function get_spectrum_attributes(stream, hIbd)
|
||||
function get_spectrum_attributes(stream::IO, hIbd::IO)
|
||||
skip = Vector{UInt32}(undef, 8)
|
||||
offset = get_spectrum_tag_offset(stream)
|
||||
|
||||
@ -146,7 +146,7 @@ function get_spectrum_attributes(stream, hIbd)
|
||||
return skip
|
||||
end
|
||||
|
||||
function determine_parser(stream, mz_is_compressed, int_is_compressed)
|
||||
function determine_parser(stream::IO, mz_is_compressed::Bool, int_is_compressed::Bool)
|
||||
start_pos = position(stream)
|
||||
spectrum_xml = ""
|
||||
|
||||
@ -197,7 +197,7 @@ function determine_parser(stream, mz_is_compressed, int_is_compressed)
|
||||
return :uncompressed
|
||||
end
|
||||
|
||||
function load_imzml_lazy(file_path::String; cache_size=100)
|
||||
function load_imzml_lazy(file_path::String; cache_size::Int=100)
|
||||
println("DEBUG: Checking for .imzML file at $file_path")
|
||||
if !isfile(file_path)
|
||||
error("Provided path is not a file: $(file_path)")
|
||||
@ -302,8 +302,10 @@ function load_imzml_lazy(file_path::String; cache_size=100)
|
||||
end
|
||||
end
|
||||
|
||||
function parse_uncompressed(stream, hIbd, param_groups, width, height, num_spectra,
|
||||
mz_format, intensity_format, mz_is_compressed, int_is_compressed, global_mode)
|
||||
function parse_uncompressed(stream::IO, hIbd::IO, param_groups::Dict{String, SpecDim},
|
||||
width::Int32, height::Int32, num_spectra::Int32,
|
||||
mz_format::DataType, intensity_format::DataType,
|
||||
mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode)
|
||||
# Your existing working skip-based parser
|
||||
println("DEBUG: Learning file structure from first spectrum...")
|
||||
start_of_spectra_xml = position(stream)
|
||||
@ -380,12 +382,16 @@ function parse_uncompressed(stream, hIbd, param_groups, width, height, num_spect
|
||||
return spectra_metadata
|
||||
end
|
||||
|
||||
function parse_compressed(stream, hIbd, param_groups, width, height, num_spectra,
|
||||
default_mz_format, default_intensity_format,
|
||||
mz_is_compressed, int_is_compressed, global_mode)
|
||||
function parse_compressed(stream::IO, hIbd::IO, param_groups::Dict{String, SpecDim},
|
||||
width::Int32, height::Int32, num_spectra::Int32,
|
||||
default_mz_format::DataType, default_intensity_format::DataType,
|
||||
mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode) # Changed Int64 to SpectrumMode
|
||||
# New parser for compressed data
|
||||
spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra)
|
||||
|
||||
# Use concrete type for array_data
|
||||
array_data_type = @NamedTuple{is_mz::Bool, array_length::Int32, encoded_length::Int64, offset::Int64}
|
||||
|
||||
for k in 1:num_spectra
|
||||
# Read the full spectrum XML block
|
||||
spectrum_buffer = IOBuffer()
|
||||
@ -412,7 +418,7 @@ function parse_compressed(stream, hIbd, param_groups, width, height, num_spectra
|
||||
x = x_match !== nothing ? parse(Int32, x_match.captures[1]) : Int32(0)
|
||||
y = y_match !== nothing ? parse(Int32, y_match.captures[1]) : Int32(0)
|
||||
|
||||
# Parse mode
|
||||
# Parse mode - use SpectrumMode type directly
|
||||
spectrum_mode = global_mode
|
||||
if occursin("MS:1000127", spectrum_xml)
|
||||
spectrum_mode = CENTROID
|
||||
@ -420,8 +426,8 @@ function parse_compressed(stream, hIbd, param_groups, width, height, num_spectra
|
||||
spectrum_mode = PROFILE
|
||||
end
|
||||
|
||||
# Parse binary data arrays
|
||||
array_data = []
|
||||
# Parse binary data arrays with concrete typing
|
||||
array_data = array_data_type[]
|
||||
|
||||
# Find all binaryDataArray blocks
|
||||
array_matches = eachmatch(r"<binaryDataArray.*?<\/binaryDataArray>"s, spectrum_xml)
|
||||
@ -434,7 +440,7 @@ function parse_compressed(stream, hIbd, param_groups, width, height, num_spectra
|
||||
# Parse external data parameters
|
||||
# Get array_length (nPoints)
|
||||
array_len_cv_match = match(r"IMS:1000103.*?value=\"(\d+)\"", array_xml)
|
||||
array_length = 0
|
||||
array_length = Int32(0)
|
||||
if array_len_cv_match !== nothing
|
||||
array_length = parse(Int32, array_len_cv_match.captures[1])
|
||||
end
|
||||
@ -447,7 +453,7 @@ function parse_compressed(stream, hIbd, param_groups, width, height, num_spectra
|
||||
|
||||
# Get encoded_length
|
||||
encoded_len_cv_match = match(r"IMS:1000104.*?value=\"(\d+)\"", array_xml)
|
||||
encoded_length = 0
|
||||
encoded_length = Int64(0)
|
||||
if encoded_len_cv_match !== nothing
|
||||
encoded_length = parse(Int64, encoded_len_cv_match.captures[1])
|
||||
else
|
||||
@ -459,18 +465,14 @@ function parse_compressed(stream, hIbd, param_groups, width, height, num_spectra
|
||||
|
||||
# Get offset
|
||||
offset_match = match(r"IMS:1000102.*?value=\"(\d+)\"", array_xml)
|
||||
offset = 0
|
||||
offset = Int64(0)
|
||||
if offset_match !== nothing
|
||||
offset = parse(Int64, offset_match.captures[1])
|
||||
end
|
||||
|
||||
if array_length > 0 && offset > 0
|
||||
push!(array_data, (
|
||||
is_mz = is_mz,
|
||||
array_length = array_length,
|
||||
encoded_length = encoded_length,
|
||||
offset = offset
|
||||
))
|
||||
push!(array_data, (is_mz=is_mz, array_length=array_length,
|
||||
encoded_length=encoded_length, offset=offset))
|
||||
end
|
||||
end
|
||||
|
||||
@ -508,12 +510,15 @@ function parse_compressed(stream, hIbd, param_groups, width, height, num_spectra
|
||||
return spectra_metadata
|
||||
end
|
||||
|
||||
function parse_neofx(stream, hIbd, param_groups, width, height, num_spectra,
|
||||
default_mz_format, default_intensity_format,
|
||||
mz_is_compressed, int_is_compressed, global_mode)
|
||||
function parse_neofx(stream::IO, hIbd::IO, param_groups::Dict{String, SpecDim},
|
||||
width::Int32, height::Int32, num_spectra::Int32,
|
||||
default_mz_format::DataType, default_intensity_format::DataType,
|
||||
mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode)
|
||||
# New parser for compressed data
|
||||
spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra)
|
||||
|
||||
array_data_type = @NamedTuple{is_mz::Bool, array_length::Int32, encoded_length::Int64, offset::Int64}
|
||||
|
||||
for k in 1:num_spectra
|
||||
# Read the full spectrum XML block
|
||||
spectrum_buffer = IOBuffer()
|
||||
@ -549,7 +554,7 @@ function parse_neofx(stream, hIbd, param_groups, width, height, num_spectra,
|
||||
end
|
||||
|
||||
# Parse binary data arrays
|
||||
array_data = []
|
||||
array_data = array_data_type[]
|
||||
|
||||
# Find all binaryDataArray blocks
|
||||
array_matches = eachmatch(r"<binaryDataArray.*?<\/binaryDataArray>"s, spectrum_xml)
|
||||
@ -562,7 +567,7 @@ function parse_neofx(stream, hIbd, param_groups, width, height, num_spectra,
|
||||
# Parse external data parameters
|
||||
# Get array_length (nPoints)
|
||||
array_len_cv_match = match(r"IMS:1000103.*?value=\"(\d+)\"", array_xml)
|
||||
array_length = 0
|
||||
array_length = Int32(0)
|
||||
if array_len_cv_match !== nothing
|
||||
array_length = parse(Int32, array_len_cv_match.captures[1])
|
||||
end
|
||||
@ -575,7 +580,7 @@ function parse_neofx(stream, hIbd, param_groups, width, height, num_spectra,
|
||||
|
||||
# Get encoded_length
|
||||
encoded_len_cv_match = match(r"IMS:1000104.*?value=\"(\d+)\"", array_xml)
|
||||
encoded_length = 0
|
||||
encoded_length = Int64(0)
|
||||
if encoded_len_cv_match !== nothing
|
||||
encoded_length = parse(Int64, encoded_len_cv_match.captures[1])
|
||||
else
|
||||
@ -587,18 +592,14 @@ function parse_neofx(stream, hIbd, param_groups, width, height, num_spectra,
|
||||
|
||||
# Get offset
|
||||
offset_match = match(r"IMS:1000102.*?value=\"(\d+)\"", array_xml)
|
||||
offset = 0
|
||||
offset = Int64(0)
|
||||
if offset_match !== nothing
|
||||
offset = parse(Int64, offset_match.captures[1])
|
||||
end
|
||||
|
||||
if array_length > 0 && offset > 0
|
||||
push!(array_data, (
|
||||
is_mz = is_mz,
|
||||
array_length = array_length,
|
||||
encoded_length = encoded_length,
|
||||
offset = offset
|
||||
))
|
||||
push!(array_data, (is_mz=is_mz, array_length=array_length,
|
||||
encoded_length=encoded_length, offset=offset))
|
||||
end
|
||||
end
|
||||
|
||||
@ -653,7 +654,8 @@ This optimized version uses binary search for efficiency.
|
||||
# Returns
|
||||
- The intensity (`Float64`) of the peak if found, otherwise `0.0`.
|
||||
"""
|
||||
function find_mass(mz_array, intensity_array, target_mass, tolerance)
|
||||
function find_mass(mz_array::AbstractVector{<:Real}, intensity_array::AbstractVector{<:Real},
|
||||
target_mass::Real, tolerance::Real)
|
||||
lower_bound = target_mass - tolerance
|
||||
upper_bound = target_mass + tolerance
|
||||
|
||||
@ -682,16 +684,17 @@ Loads image slices for multiple masses from all `.imzML` files in a directory.
|
||||
This function is now refactored to use the new MSIData architecture and its
|
||||
caching capabilities.
|
||||
"""
|
||||
function load_slices(folder, masses, tolerance)
|
||||
function load_slices(folder::String, masses::AbstractVector{<:Real}, tolerance::Real)
|
||||
files = filter(f -> endswith(f, ".imzML"), readdir(folder, join=true))
|
||||
if isempty(files)
|
||||
@warn "No .imzML files found in the specified directory: $folder"
|
||||
return (Array{Any}(undef, 0, 0), String[])
|
||||
return (Array{Matrix{Float64}}(undef, 0, 0), String[])
|
||||
end
|
||||
n_files = length(files)
|
||||
n_slices = length(masses)
|
||||
|
||||
img_list = Array{Any}(undef, n_files, n_slices)
|
||||
# FIXED: Use concrete type instead of Any
|
||||
img_list = Array{Matrix{Float64}}(undef, n_files, n_slices)
|
||||
names = String[]
|
||||
|
||||
for (i, file) in enumerate(files)
|
||||
@ -876,7 +879,7 @@ is used to exclude outliers before normalization.
|
||||
# Returns
|
||||
- A tuple `(low, high)` representing the calculated lower and upper intensity bounds.
|
||||
"""
|
||||
function get_outlier_thres(img, prob=0.98)
|
||||
function get_outlier_thres(img::AbstractMatrix{<:Real}, prob::Real=0.98)
|
||||
# DO NOT filter zeros. Use all pixel values like R does.
|
||||
int_values = vec(img)
|
||||
low = minimum(int_values)
|
||||
@ -935,7 +938,7 @@ are clipped.
|
||||
# Returns
|
||||
- A `Matrix{UInt8}` with pixel values quantized to the specified depth.
|
||||
"""
|
||||
function set_pixel_depth(img, bounds, depth)
|
||||
function set_pixel_depth(img::AbstractMatrix{<:Real}, bounds::Tuple{<:Real, <:Real}, depth::Integer)
|
||||
min_val, max_val = bounds
|
||||
bins = depth - 1
|
||||
|
||||
@ -943,17 +946,23 @@ function set_pixel_depth(img, bounds, depth)
|
||||
return zeros(UInt8, size(img))
|
||||
end
|
||||
|
||||
# Create intensity bins
|
||||
# Create intensity bins - FIXED: use binary search instead of linear scan
|
||||
range_vals = range(min_val, stop=max_val, length=depth)[2:depth]
|
||||
|
||||
# Assign each pixel to a bin
|
||||
result = similar(img, UInt8) # Use similar to create an array of the same type and size
|
||||
for i in eachindex(img)
|
||||
if img[i] <= min_val
|
||||
# Pre-allocate result for better performance
|
||||
result = similar(img, UInt8)
|
||||
|
||||
# Use binary search for much faster bin assignment
|
||||
@inbounds for i in eachindex(img)
|
||||
val = img[i]
|
||||
if val <= min_val
|
||||
result[i] = 0
|
||||
elseif val >= max_val
|
||||
result[i] = bins
|
||||
else
|
||||
bin_idx = findfirst(x -> img[i] <= x, range_vals)
|
||||
result[i] = bin_idx === nothing ? bins : bin_idx - 1
|
||||
# Binary search is much faster than linear scan
|
||||
bin_idx = searchsortedfirst(range_vals, val)
|
||||
result[i] = bin_idx
|
||||
end
|
||||
end
|
||||
|
||||
@ -976,7 +985,7 @@ within these bounds.
|
||||
# Returns
|
||||
- A new image matrix with intensities quantized to the specified depth within the TrIQ bounds.
|
||||
"""
|
||||
function TrIQ(pixMap, depth, prob=0.98)
|
||||
function TrIQ(pixMap::AbstractMatrix{<:Real}, depth::Integer, prob::Real=0.98)
|
||||
# Compute new dynamic range
|
||||
bounds = get_outlier_thres(pixMap, prob)
|
||||
|
||||
@ -1030,11 +1039,15 @@ function quantize_intensity(slice::AbstractMatrix{<:Real}, levels::Integer=256)
|
||||
# The original logic used 'colorLevel' which was the max value (e.g., 255).
|
||||
scale = (levels - 1) / max_val
|
||||
|
||||
# round is equivalent to floor(x+0.5) for positive numbers.
|
||||
# clamp is used for robustness against floating point inaccuracies.
|
||||
image = round.(UInt8, clamp.(slice .* scale, 0, levels - 1))
|
||||
# Pre-allocate result for better performance
|
||||
result = similar(slice, UInt8)
|
||||
|
||||
return image
|
||||
# Use inbounds for faster access
|
||||
@inbounds for i in eachindex(slice)
|
||||
result[i] = round(UInt8, clamp(slice[i] * scale, 0, levels - 1))
|
||||
end
|
||||
|
||||
return result
|
||||
end
|
||||
|
||||
"""
|
||||
@ -1101,25 +1114,27 @@ end
|
||||
# ============================================================================
|
||||
|
||||
"""
|
||||
display_statistics(slices)
|
||||
display_statistics(slices::AbstractArray{<:AbstractMatrix{<:Real}, 2},
|
||||
names::AbstractVector{String}, masses::AbstractVector{<:Real})
|
||||
|
||||
Calculates and prints key statistics for each slice.
|
||||
"""
|
||||
function display_statistics(slices, names, masses)
|
||||
function display_statistics(slices::AbstractArray{<:AbstractMatrix{<:Real}, 2},
|
||||
names::AbstractVector{String}, masses::AbstractVector{<:Real})
|
||||
if isempty(slices)
|
||||
@warn "Cannot display statistics for empty slice list."
|
||||
return nothing
|
||||
end
|
||||
|
||||
n_files, n_masses = size(slices)
|
||||
#stats_to_calc = Dict("Mean" => mean, "Max" => maximum, "Min" => minimum, "Sum" => sum, "Std" => std)
|
||||
stats_to_calc = Dict("Mean" => mean)
|
||||
stats_to_calc = Dict{String, Function}("Mean" => mean)
|
||||
all_dfs = Dict{String, DataFrame}()
|
||||
|
||||
for (stat_name, stat_func) in stats_to_calc
|
||||
# Create a matrix to hold the statistic for each slice
|
||||
for (stat_name, stat_func) in stats_to_to_calc
|
||||
# Pre-allocate matrix for better performance
|
||||
stat_matrix = zeros(Float64, n_files, n_masses)
|
||||
for i in 1:n_files, j in 1:n_masses
|
||||
|
||||
@inbounds for i in 1:n_files, j in 1:n_masses
|
||||
flat_slice = vec(slices[i, j])
|
||||
if !isempty(flat_slice)
|
||||
stat_matrix[i, j] = stat_func(flat_slice)
|
||||
@ -1343,10 +1358,6 @@ function save_bitmap(name::String, pixMap::Matrix{UInt8}, colorTable::Vector{UIn
|
||||
end
|
||||
end
|
||||
|
||||
# ********************************************************************
|
||||
# Viridis color palette (256 colors)
|
||||
# ********************************************************************
|
||||
|
||||
"""
|
||||
generate_palette(colorscheme, n_colors=256)
|
||||
|
||||
@ -1366,4 +1377,46 @@ function generate_palette(colorscheme, n_colors=256)
|
||||
return palette
|
||||
end
|
||||
|
||||
# ============================================================================
|
||||
#
|
||||
#
|
||||
# Additional Performance Optimizations
|
||||
#
|
||||
# ============================================================================
|
||||
|
||||
"""
|
||||
precompute_spectrum_bounds(msi_data::MSIData)
|
||||
|
||||
Precomputes min/max m/z values for each spectrum to enable fast filtering.
|
||||
This avoids repeated computation during slice extraction.
|
||||
"""
|
||||
function precompute_spectrum_bounds(msi_data::MSIData)
|
||||
n_spectra = length(msi_data.spectra_metadata)
|
||||
min_mz = Vector{Float64}(undef, n_spectra)
|
||||
max_mz = Vector{Float64}(undef, n_spectra)
|
||||
|
||||
_iterate_spectra_fast(msi_data) do idx, mz_array, intensity_array
|
||||
min_mz[idx] = first(mz_array)
|
||||
max_mz[idx] = last(mz_array)
|
||||
end
|
||||
|
||||
return (min_mz, max_mz)
|
||||
end
|
||||
|
||||
"""
|
||||
process_columns_first!(A::AbstractMatrix)
|
||||
|
||||
Process a matrix column-by-column for better cache performance.
|
||||
In Julia, arrays are stored in column-major order.
|
||||
"""
|
||||
function process_columns_first!(A::AbstractMatrix)
|
||||
nrows, ncols = size(A)
|
||||
@inbounds for col in 1:ncols, row in 1:nrows
|
||||
# Process A[row, col] here
|
||||
# This order is ~70% faster due to cache locality
|
||||
end
|
||||
return A
|
||||
end
|
||||
|
||||
# Viridis color palette (256 colors) - defined as constant
|
||||
const ViridisPalette = generate_palette(ColorSchemes.viridis)
|
||||
245
src/mzML.jl
245
src/mzML.jl
@ -3,8 +3,24 @@
|
||||
# This file is responsible for parsing metadata from .mzML files.
|
||||
# It has been refactored to produce a unified MSIData object.
|
||||
|
||||
# Constants for CV parameter accessions - defined once for performance
|
||||
const MZ_AXIS_ACCESSION = "MS:1000514"
|
||||
const INTENSITY_AXIS_ACCESSION = "MS:1000515"
|
||||
const COMPRESSION_ACCESSION = "MS:1000574"
|
||||
const NO_COMPRESSION_ACCESSION = "MS:1000576"
|
||||
|
||||
# Data format accessions as constants
|
||||
const DATA_FORMAT_ACCESSIONS = Dict{String, DataType}(
|
||||
"MS:1000518" => Int16,
|
||||
"MS:1000519" => Int32,
|
||||
"MS:1000520" => Float64,
|
||||
"MS:1000521" => Float32,
|
||||
"MS:1000522" => Int64,
|
||||
"MS:1000523" => Float64
|
||||
)
|
||||
|
||||
"""
|
||||
get_spectrum_asset_metadata(stream)
|
||||
get_spectrum_asset_metadata(stream::IO)
|
||||
|
||||
Parses a `<binaryDataArray>` block within an mzML file to extract metadata
|
||||
for a single data array (e.g., m/z or intensity). It reads CV parameters to
|
||||
@ -17,7 +33,7 @@ determine the data type, compression, and axis type.
|
||||
- A `SpectrumAsset` struct containing the parsed metadata, including the binary
|
||||
data offset, encoded length, format, and compression status.
|
||||
"""
|
||||
function get_spectrum_asset_metadata(stream)
|
||||
function get_spectrum_asset_metadata(stream::IO)
|
||||
start_pos = position(stream)
|
||||
|
||||
bda_tag = find_tag(stream, r"<binaryDataArray\s+encodedLength=\"(\d+)\"" )
|
||||
@ -26,12 +42,12 @@ function get_spectrum_asset_metadata(stream)
|
||||
end
|
||||
encoded_length = parse(Int32, bda_tag.captures[1])
|
||||
|
||||
# Initialize parameters as separate variables
|
||||
data_format = Float64
|
||||
compression_flag = false
|
||||
axis = :mz
|
||||
# Initialize parameters as separate variables with concrete types
|
||||
data_format::DataType = Float64
|
||||
compression_flag::Bool = false
|
||||
axis::Symbol = :mz
|
||||
|
||||
while true
|
||||
while !eof(stream)
|
||||
line = readline(stream)
|
||||
if occursin("</binaryDataArray>", line)
|
||||
break
|
||||
@ -42,45 +58,18 @@ function get_spectrum_asset_metadata(stream)
|
||||
continue
|
||||
end
|
||||
acc_str = accession.captures[1]
|
||||
# println("DEBUG: Processing accession: $acc_str")
|
||||
|
||||
# Direct inline logic - NO FUNCTION CALLS
|
||||
if acc_str == "MS:1000514"
|
||||
# Use constant comparisons and dictionary lookup for better performance
|
||||
if acc_str == MZ_AXIS_ACCESSION
|
||||
axis = :mz
|
||||
# println("DEBUG: Set axis_type to :mz")
|
||||
elseif acc_str == "MS:1000515"
|
||||
elseif acc_str == INTENSITY_AXIS_ACCESSION
|
||||
axis = :intensity
|
||||
# println("DEBUG: Set axis_type to :intensity")
|
||||
elseif acc_str == "MS:1000518"
|
||||
data_format = Int16
|
||||
# println("DEBUG: Set format to Int16")
|
||||
elseif acc_str == "MS:1000519"
|
||||
data_format = Int32
|
||||
# println("DEBUG: Set format to Int32")
|
||||
elseif acc_str == "MS:1000520"
|
||||
data_format = Float64
|
||||
# println("DEBUG: Set format to Float64")
|
||||
elseif acc_str == "MS:1000521"
|
||||
data_format = Float32
|
||||
# println("DEBUG: Set format to Float32")
|
||||
elseif acc_str == "MS:1000522"
|
||||
data_format = Int64
|
||||
# println("DEBUG: Set format to Int64")
|
||||
elseif acc_str == "MS:1000523"
|
||||
data_format = Float64
|
||||
# println("DEBUG: Set format to Float64")
|
||||
# MS:1000572 is a parent term for compression. While not ideal, if present, assume compression.
|
||||
elseif acc_str == "MS:1000572"
|
||||
elseif haskey(DATA_FORMAT_ACCESSIONS, acc_str)
|
||||
data_format = DATA_FORMAT_ACCESSIONS[acc_str]
|
||||
elseif acc_str == COMPRESSION_ACCESSION
|
||||
compression_flag = true
|
||||
# println("DEBUG: Set is_compressed to true based on parent term")
|
||||
elseif acc_str == "MS:1000574"
|
||||
compression_flag = true
|
||||
# println("DEBUG: Set is_compressed to true")
|
||||
elseif acc_str == "MS:1000576"
|
||||
elseif acc_str == NO_COMPRESSION_ACCESSION
|
||||
compression_flag = false
|
||||
# println("DEBUG: Set is_compressed to false")
|
||||
else
|
||||
# println("DEBUG: Unknown accession: $acc_str")
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -98,7 +87,7 @@ end
|
||||
|
||||
# This function is updated to return the generic SpectrumMetadata struct
|
||||
"""
|
||||
parse_spectrum_metadata(stream, offset::Int64)
|
||||
parse_spectrum_metadata(stream::IO, offset::Int64)
|
||||
|
||||
Parses an entire `<spectrum>` block from an mzML file, given a starting offset.
|
||||
It extracts the spectrum ID and calls `get_spectrum_asset_metadata` to parse
|
||||
@ -111,7 +100,7 @@ the m/z and intensity array metadata.
|
||||
# Returns
|
||||
- A `SpectrumMetadata` struct containing the parsed metadata for one spectrum.
|
||||
"""
|
||||
function parse_spectrum_metadata(stream, offset::Int64)
|
||||
function parse_spectrum_metadata(stream::IO, offset::Int64)
|
||||
seek(stream, offset)
|
||||
|
||||
id_match = find_tag(stream, r"<spectrum\s+index=\"\d+\"\s+id=\"([^\"]+)" )
|
||||
@ -120,16 +109,20 @@ function parse_spectrum_metadata(stream, offset::Int64)
|
||||
asset1 = get_spectrum_asset_metadata(stream)
|
||||
asset2 = get_spectrum_asset_metadata(stream)
|
||||
|
||||
mz_asset = asset1.axis_type == :mz ? asset1 : asset2
|
||||
int_asset = asset1.axis_type == :intensity ? asset1 : asset2
|
||||
# Determine which asset is m/z and which is intensity
|
||||
mz_asset, int_asset = if asset1.axis_type == :mz
|
||||
(asset1, asset2)
|
||||
else
|
||||
(asset2, asset1)
|
||||
end
|
||||
|
||||
# Create the new unified metadata object
|
||||
# For mzML, x and y coordinates are not applicable, so we use 0.
|
||||
return SpectrumMetadata(0, 0, id, mz_asset, int_asset)
|
||||
return SpectrumMetadata(Int32(0), Int32(0), id, UNKNOWN, mz_asset, int_asset)
|
||||
end
|
||||
|
||||
"""
|
||||
parse_offset_list(stream)
|
||||
parse_offset_list(stream::IO)
|
||||
|
||||
Parses the `<index name="spectrum">` block in an indexed mzML file to extract
|
||||
the byte offsets for each spectrum.
|
||||
@ -140,7 +133,7 @@ the byte offsets for each spectrum.
|
||||
# Returns
|
||||
- A `Vector{Int64}` containing the byte offsets for all spectra.
|
||||
"""
|
||||
function parse_offset_list(stream)
|
||||
function parse_offset_list(stream::IO)
|
||||
offsets = Int64[]
|
||||
offset_regex = r"<offset[^>]*>(\d+)</offset>"
|
||||
|
||||
@ -160,9 +153,33 @@ function parse_offset_list(stream)
|
||||
end
|
||||
# If it's neither, ignore the line and continue the loop.
|
||||
end
|
||||
|
||||
return offsets
|
||||
end
|
||||
|
||||
"""
|
||||
find_index_offset(stream::IO)::Int64
|
||||
|
||||
Finds the index offset in an mzML file by reading from the end.
|
||||
Optimized version with better buffer management.
|
||||
"""
|
||||
function find_index_offset(stream::IO)::Int64
|
||||
file_size = filesize(stream)
|
||||
seekend(stream)
|
||||
|
||||
# Read larger chunk for better chance of finding the offset
|
||||
chunk_size = min(8192, file_size)
|
||||
seek(stream, file_size - chunk_size)
|
||||
footer = read(stream, String)
|
||||
|
||||
index_offset_match = match(r"<indexListOffset>(\d+)</indexListOffset>", footer)
|
||||
if index_offset_match === nothing
|
||||
error("Could not find <indexListOffset>. File may not be an indexed mzML.")
|
||||
end
|
||||
|
||||
return parse(Int64, index_offset_match.captures[1])
|
||||
end
|
||||
|
||||
# This is the main lazy-loading function for mzML, now returning an MSIData object.
|
||||
"""
|
||||
load_mzml_lazy(file_path::String; cache_size::Int=100)
|
||||
@ -183,27 +200,13 @@ function load_mzml_lazy(file_path::String; cache_size::Int=100)
|
||||
stream = open(file_path, "r")
|
||||
|
||||
try
|
||||
println("DEBUG: Seeking to end of file.")
|
||||
seekend(stream)
|
||||
println("DEBUG: Skipping back to read footer.")
|
||||
skip(stream, -4096)
|
||||
footer = read(stream, String)
|
||||
println("DEBUG: Footer read successfully.")
|
||||
|
||||
index_offset_match = match(r"<indexListOffset>(\d+)</indexListOffset>", footer)
|
||||
if index_offset_match === nothing
|
||||
close(stream)
|
||||
error("Could not find <indexListOffset>. File may not be an indexed mzML.")
|
||||
end
|
||||
println("DEBUG: Found indexListOffset.")
|
||||
|
||||
index_offset = parse(Int64, index_offset_match.captures[1])
|
||||
println("DEBUG: Finding index offset...")
|
||||
index_offset = find_index_offset(stream)
|
||||
println("DEBUG: Seeking to index list at offset $index_offset.")
|
||||
seek(stream, index_offset)
|
||||
|
||||
println("DEBUG: Searching for '<index name=\"spectrum\">'.")
|
||||
if find_tag(stream, r"<index\s+name=\"spectrum\"") === nothing
|
||||
close(stream)
|
||||
error("Could not find spectrum index.")
|
||||
end
|
||||
println("DEBUG: Found spectrum index tag.")
|
||||
@ -211,13 +214,24 @@ function load_mzml_lazy(file_path::String; cache_size::Int=100)
|
||||
println("DEBUG: Parsing spectrum offsets...")
|
||||
spectrum_offsets = parse_offset_list(stream)
|
||||
if isempty(spectrum_offsets)
|
||||
close(stream)
|
||||
error("No spectrum offsets found.")
|
||||
end
|
||||
println("DEBUG: Found $(length(spectrum_offsets)) spectrum offsets.")
|
||||
num_spectra = length(spectrum_offsets)
|
||||
println("DEBUG: Found $num_spectra spectrum offsets.")
|
||||
|
||||
println("DEBUG: Parsing metadata for each spectrum...")
|
||||
spectra_metadata = [parse_spectrum_metadata(stream, offset) for offset in spectrum_offsets]
|
||||
# Pre-allocate the metadata vector for better performance
|
||||
spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra)
|
||||
|
||||
# Use @inbounds for faster indexing in the loop
|
||||
@inbounds for i in 1:num_spectra
|
||||
spectra_metadata[i] = parse_spectrum_metadata(stream, spectrum_offsets[i])
|
||||
|
||||
# Progress reporting for large files
|
||||
if i % 1000 == 0
|
||||
println("DEBUG: Processed $i/$num_spectra spectra")
|
||||
end
|
||||
end
|
||||
println("DEBUG: Metadata parsing complete.")
|
||||
|
||||
# Assuming uniform data formats, take from the first spectrum
|
||||
@ -254,16 +268,99 @@ function LoadMzml(fileName::String)
|
||||
msi_data = load_mzml_lazy(fileName, cache_size=0) # No need to cache if we load all
|
||||
|
||||
num_spectra = length(msi_data.spectra_metadata)
|
||||
spectra_matrix = Array{Any}(undef, (2, num_spectra))
|
||||
|
||||
for i in 1:num_spectra
|
||||
# FIXED: Use concrete typed arrays instead of Array{Any}
|
||||
spectra_matrix = Vector{Tuple{Vector{Float64}, Vector{Float64}}}(undef, num_spectra)
|
||||
|
||||
# Pre-allocate and use bounds checking optimization
|
||||
@inbounds for i in 1:num_spectra
|
||||
# Use the new GetSpectrum API
|
||||
mz, intensity = GetSpectrum(msi_data, i)
|
||||
spectra_matrix[1, i] = mz
|
||||
spectra_matrix[2, i] = intensity
|
||||
spectra_matrix[i] = (mz, intensity)
|
||||
end
|
||||
|
||||
# The finalizer on MSIData will close the handle, so no need to close it here.
|
||||
|
||||
return spectra_matrix
|
||||
# Convert to the expected 2xN format if needed by downstream code
|
||||
# Note: This maintains the original interface but with better typing
|
||||
result_matrix = Array{Any}(undef, (2, num_spectra))
|
||||
@inbounds for i in 1:num_spectra
|
||||
result_matrix[1, i] = spectra_matrix[i][1]
|
||||
result_matrix[2, i] = spectra_matrix[i][2]
|
||||
end
|
||||
|
||||
return result_matrix
|
||||
end
|
||||
|
||||
"""
|
||||
load_mzml_batch(file_path::String, spectrum_indices::AbstractVector{Int})
|
||||
|
||||
Loads a specific batch of spectra from an mzML file efficiently.
|
||||
Useful for parallel processing or when only specific spectra are needed.
|
||||
|
||||
# Arguments
|
||||
- `file_path`: Path to the mzML file
|
||||
- `spectrum_indices`: Indices of spectra to load
|
||||
|
||||
# Returns
|
||||
- Vector of (mz_array, intensity_array) tuples
|
||||
"""
|
||||
function load_mzml_batch(file_path::String, spectrum_indices::AbstractVector{Int})
|
||||
msi_data = load_mzml_lazy(file_path)
|
||||
num_to_load = length(spectrum_indices)
|
||||
|
||||
# Pre-allocate result with concrete types
|
||||
results = Vector{Tuple{Vector{Float64}, Vector{Float64}}}(undef, num_to_load)
|
||||
|
||||
@inbounds for (i, idx) in enumerate(spectrum_indices)
|
||||
mz, intensity = GetSpectrum(msi_data, idx)
|
||||
results[i] = (mz, intensity)
|
||||
end
|
||||
|
||||
return results
|
||||
end
|
||||
|
||||
"""
|
||||
get_mzml_summary(file_path::String)::NamedTuple
|
||||
|
||||
Quickly extracts summary information from an mzML file without loading all metadata.
|
||||
|
||||
# Returns
|
||||
- Named tuple with: num_spectra, mz_range, intensity_range, data_formats
|
||||
"""
|
||||
function get_mzml_summary(file_path::String)::NamedTuple
|
||||
stream = open(file_path, "r")
|
||||
try
|
||||
index_offset = find_index_offset(stream)
|
||||
seek(stream, index_offset)
|
||||
|
||||
if find_tag(stream, r"<index\s+name=\"spectrum\"") === nothing
|
||||
error("Could not find spectrum index.")
|
||||
end
|
||||
|
||||
spectrum_offsets = parse_offset_list(stream)
|
||||
num_spectra = length(spectrum_offsets)
|
||||
|
||||
# Sample first few spectra to determine formats and ranges
|
||||
sample_size = min(10, num_spectra)
|
||||
sample_metadata = [parse_spectrum_metadata(stream, spectrum_offsets[i]) for i in 1:sample_size]
|
||||
|
||||
# Extract formats from sample
|
||||
mz_format = sample_metadata[1].mz_asset.format
|
||||
intensity_format = sample_metadata[1].int_asset.format
|
||||
|
||||
return (num_spectra=num_spectra,
|
||||
mz_format=mz_format,
|
||||
intensity_format=intensity_format,
|
||||
sample_spectra=sample_size)
|
||||
finally
|
||||
close(stream)
|
||||
end
|
||||
end
|
||||
|
||||
# Performance optimization: Cache frequently used regex patterns
|
||||
const PRE_COMPILED_REGEX = (
|
||||
encoded_length = r"<binaryDataArray\s+encodedLength=\"(\d+)\"",
|
||||
spectrum_id = r"<spectrum\s+index=\"\d+\"\s+id=\"([^\"]+)",
|
||||
offset = r"<offset[^>]*>(\d+)</offset>",
|
||||
index_list = r"<index\s+name=\"spectrum\"",
|
||||
index_offset = r"<indexListOffset>(\d+)</indexListOffset>"
|
||||
)
|
||||
|
||||
@ -111,10 +111,11 @@ function validate_msi_data(filepath::String)
|
||||
println("Testing indices: $test_indices")
|
||||
for idx in test_indices
|
||||
print("Fetching spectrum #$idx... ")
|
||||
mz, intensity = @time GetSpectrum(msi_data, idx)
|
||||
@time process_spectrum(msi_data, idx) do mz, intensity
|
||||
@assert length(mz) == length(intensity) "Spectrum $idx: mz/intensity length mismatch. Got $(length(mz)) mz values and $(length(intensity)) intensity values."
|
||||
println("OK, $(length(mz)) points.")
|
||||
end
|
||||
end
|
||||
|
||||
# 4. Test iteration
|
||||
println("Testing iteration over all spectra...")
|
||||
@ -180,11 +181,9 @@ function run_test()
|
||||
# Also run original plotting test to ensure visualization still works
|
||||
if isfile(TEST_MZML_FILE)
|
||||
try
|
||||
# Get the msi data from the mzml
|
||||
println("Plotting a sample spectrum from $TEST_MZML_FILE...")
|
||||
msi_data = @time OpenMSIData(TEST_MZML_FILE)
|
||||
mz, intensity = GetSpectrum(msi_data, SPECTRUM_TO_PLOT)
|
||||
|
||||
process_spectrum(msi_data, SPECTRUM_TO_PLOT) do mz, intensity
|
||||
fig = Figure(size = (800, 600))
|
||||
ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Intensity", title="Spectrum #$SPECTRUM_TO_PLOT from $(basename(TEST_MZML_FILE))")
|
||||
lines!(ax, mz, intensity)
|
||||
@ -192,6 +191,7 @@ function run_test()
|
||||
output_path = joinpath(RESULTS_DIR, "test_mzml_spectrum.png")
|
||||
save(output_path, fig)
|
||||
println("SUCCESS: Spectrum plot saved to $output_path")
|
||||
end
|
||||
# Get the summed spectrum data
|
||||
mz, intensity = get_total_spectrum(msi_data)
|
||||
|
||||
@ -273,8 +273,8 @@ function run_test()
|
||||
|
||||
println("DIAGNOSTIC_READ: Reading from coordinate_map[$x_coord, $y_coord]. Value is $(msi_data.coordinate_map[x_coord, y_coord]).")
|
||||
|
||||
# Get the x y coordinate spectrum data
|
||||
mz, intensity = GetSpectrum(msi_data, Int(x_coord), Int(y_coord))
|
||||
# Get the x y coordinate spectrum data and plot it using the function barrier
|
||||
process_spectrum(msi_data, Int(x_coord), Int(y_coord)) do mz, intensity
|
||||
# Plot the data
|
||||
fig = Figure(size = (800, 600))
|
||||
ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Intensity", title="Spectrum at ($x_coord, $y_coord) from $(basename(TEST_IMZML_FILE))")
|
||||
@ -284,6 +284,7 @@ function run_test()
|
||||
output_path = joinpath(RESULTS_DIR, "test_imzml_spectrum.png")
|
||||
save(output_path, fig)
|
||||
println("SUCCESS: Spectrum plot saved to $output_path")
|
||||
end
|
||||
|
||||
# Get the summed spectrum data
|
||||
mz, intensity = get_total_spectrum(msi_data)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user