Julian/JuliaMSI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed preprocessing module, implemented precalculations, this code gives a prediction of the values for the parameters in every step for preprocessing, added test suites for both the parameter generation and the preprocessing pipeline, included a descriptive processing that returns an image with what every parameter does in every step, added instrument metadata structure with the parser for each type of file, to retrieve several important parameters to automate precalculations and preprocessing

Browse Source
This commit is contained in:
Pixelguy14 2025-11-25 12:53:57 -06:00
parent 8fa92dc0e5
commit bdd9315796
15 changed files with 4167 additions and 1676 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Manifest.toml
View File

@ -2,7 +2,7 @@
julia_version = "1.11.7" julia_version = "1.11.7"
manifest_format = "2.0" manifest_format = "2.0"
project_hash = "7843f141174fe17a3abc4f25c3819d3195459b5a" project_hash = "4955dfdbb7a233ddefa69e3cce2833f416ffe1bc"
[[deps.ANSIColoredPrinters]] [[deps.ANSIColoredPrinters]]
git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c"

2
Project.toml
View File

@ -35,11 +35,13 @@ Loess = "4345ca2d-374a-55d4-8d30-97f9976e7612"
Mmap = "a63ad114-7e13-5084-954f-fe012c677804" Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4" NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4"
NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85" NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85"
Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a"
PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5" PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
SavitzkyGolay = "c4bf5708-b6a6-4fbe-bcd0-6850ed671584" SavitzkyGolay = "c4bf5708-b6a6-4fbe-bcd0-6850ed671584"
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
StipplePlotly = "ec984513-233d-481d-95b0-a3b58b97af2b" StipplePlotly = "ec984513-233d-481d-95b0-a3b58b97af2b"

167
src/MSIData.jl
View File

@ -6,7 +6,7 @@ including caching and iteration logic, for handling large mzML and imzML dataset
efficiently. efficiently.
""" """
using Base64, Libz, Serialization, Printf, DataFrames, Base.Threads # For reading binary data using Base64, Libz, Serialization, Printf, DataFrames, Base.Threads, StatsBase # For reading binary data
const FILE_HANDLE_LOCK = ReentrantLock() const FILE_HANDLE_LOCK = ReentrantLock()
@ -28,9 +28,16 @@ function ThreadSafeFileHandle(path::String, mode::String="r")
return ThreadSafeFileHandle(path, handle, ReentrantLock()) return ThreadSafeFileHandle(path, handle, ReentrantLock())
end end
function Base.seek(tsfh::ThreadSafeFileHandle, pos::Integer) """
read_at!(tsfh::ThreadSafeFileHandle, a::AbstractArray, pos::Integer)
Atomically seeks to a position and reads data into an array. This is the thread-safe
way to read from a specific offset in the file.
"""
function read_at!(tsfh::ThreadSafeFileHandle, a::AbstractArray, pos::Integer)
lock(tsfh.lock) do lock(tsfh.lock) do
seek(tsfh.handle, pos) seek(tsfh.handle, pos)
read!(tsfh.handle, a)
end end
end end
@ -40,12 +47,6 @@ function Base.read(tsfh::ThreadSafeFileHandle, n::Integer)
end end
end end
function Base.read!(tsfh::ThreadSafeFileHandle, a::AbstractArray)
lock(tsfh.lock) do
return read!(tsfh.handle, a)
end
end
function Base.close(tsfh::ThreadSafeFileHandle) function Base.close(tsfh::ThreadSafeFileHandle)
close(tsfh.handle) close(tsfh.handle)
end end
@ -66,6 +67,20 @@ function Base.filesize(tsfh::ThreadSafeFileHandle)
end end
end end
# Overload Base.seek for ThreadSafeFileHandle
function Base.seek(tsfh::ThreadSafeFileHandle, pos::Integer)
lock(tsfh.lock) do
seek(tsfh.handle, pos)
end
end
# Overload Base.read! for ThreadSafeFileHandle
function Base.read!(tsfh::ThreadSafeFileHandle, a::AbstractArray)
lock(tsfh.lock) do
read!(tsfh.handle, a)
end
end
# Abstract type for different data sources (e.g., mzML, imzML) # Abstract type for different data sources (e.g., mzML, imzML)
# This allows dispatching to the correct binary reading logic. # This allows dispatching to the correct binary reading logic.
abstract type MSDataSource end abstract type MSDataSource end
@ -131,6 +146,50 @@ An enum representing the type of a spectrum's data.
""" """
@enum SpectrumMode CENTROID=1 PROFILE=2 UNKNOWN=3 @enum SpectrumMode CENTROID=1 PROFILE=2 UNKNOWN=3
"""
InstrumentMetadata
Contains metadata about the instrument configuration and acquisition parameters.
This information is parsed from the file header and is crucial for guiding
preprocessing steps.
# Fields
- `resolution`: Instrument resolution at a specific m/z, if available.
- `acquisition_mode`: The overall data acquisition mode (`:profile`, `:centroid`, or `:mixed`).
- `mz_axis_type`: The nature of the m/z axis (`:regular`, `:irregular`, `:mixed`).
- `calibration_status`: The calibration state of the data (`:internal`, `:external`, `:uncalibrated`).
- `instrument_model`: The vendor and model of the instrument.
- `mass_accuracy_ppm`: Manufacturer-specified mass accuracy in parts-per-million.
- `laser_settings`: A dictionary of MSI-specific laser parameters.
- `polarity`: The polarity of the acquisition (`:positive`, `:negative`, `:unknown`).
"""
struct InstrumentMetadata
resolution::Union{Float64, Nothing}
acquisition_mode::Symbol
mz_axis_type::Symbol
calibration_status::Symbol
instrument_model::String
mass_accuracy_ppm::Union{Float64, Nothing}
laser_settings::Union{Dict, Nothing}
polarity::Symbol
vendor_preprocessing_steps::Union{Vector{String}, Nothing} # New field for provenance
end
# Default constructor for initializing with unknown values
function InstrumentMetadata()
return InstrumentMetadata(
nothing, # resolution
:unknown, # acquisition_mode
:unknown, # mz_axis_type
:uncalibrated, # calibration_status
"Not Available",# instrument_model
nothing, # mass_accuracy_ppm
nothing, # laser_settings
:unknown, # polarity
nothing # vendor_preprocessing
)
end
""" """
SpectrumMetadata SpectrumMetadata
@ -150,6 +209,7 @@ struct SpectrumMetadata
# For mzML # For mzML
id::String id::String
type::Symbol # NEW: e.g., :sample, :blank, :qc
mode::SpectrumMode mode::SpectrumMode
# Common binary data info # Common binary data info
@ -180,6 +240,7 @@ efficient repeated access to spectra.
mutable struct MSIData mutable struct MSIData
source::MSDataSource source::MSDataSource
spectra_metadata::Vector{SpectrumMetadata} spectra_metadata::Vector{SpectrumMetadata}
instrument_metadata::Union{InstrumentMetadata, Nothing}
image_dims::Tuple{Int, Int} image_dims::Tuple{Int, Int}
coordinate_map::Union{Matrix{Int}, Nothing} coordinate_map::Union{Matrix{Int}, Nothing}
@ -198,13 +259,14 @@ mutable struct MSIData
spectrum_stats_df::Union{DataFrame, Nothing} spectrum_stats_df::Union{DataFrame, Nothing}
bloom_filters::Union{Vector{<:BloomFilter}, Nothing} bloom_filters::Union{Vector{<:BloomFilter}, Nothing}
analytics_ready::AtomicFlag analytics_ready::AtomicFlag
preprocessing_hints::Union{Dict{Symbol, Any}, Nothing} # For auto-determined parameters
function MSIData(source, metadata, dims, coordinate_map, cache_size) function MSIData(source, metadata, instrument_meta, dims, coordinate_map, cache_size)
obj = new(source, metadata, dims, coordinate_map, obj = new(source, metadata, instrument_meta, dims, coordinate_map,
Dict(), [], cache_size, ReentrantLock(), Dict(), [], cache_size, ReentrantLock(),
SimpleBufferPool(), SimpleBufferPool(),
Threads.Atomic{Float64}(Inf), Threads.Atomic{Float64}(-Inf), Threads.Atomic{Float64}(Inf), Threads.Atomic{Float64}(-Inf),
nothing, nothing, AtomicFlag()) nothing, nothing, AtomicFlag(), nothing)
# Ensure file handles are closed when the object is garbage collected # Ensure file handles are closed when the object is garbage collected
finalizer(obj) do o finalizer(obj) do o
@ -407,16 +469,17 @@ function read_binary_vector(data::MSIData, io::IO, asset::SpectrumAsset)
# Calculate number of elements # Calculate number of elements
n_elements = length(decoded_bytes) ÷ sizeof(asset.format) n_elements = length(decoded_bytes) ÷ sizeof(asset.format)
out_array = Vector{asset.format}(undef, n_elements)
# Use unsafe_wrap to avoid copy - this is the critical optimization if n_elements * sizeof(asset.format) != length(decoded_bytes)
temp_array = unsafe_wrap(Vector{asset.format}, pointer(decoded_bytes), n_elements) throw(FileFormatError("Size of decoded byte array is not a multiple of the element size."))
# Convert byte order in-place
@inbounds for i in 1:n_elements
out_array[i] = ltoh(temp_array[i])
end end
# Reinterpret the byte array as an array of the target type. This does not copy.
reinterpreted_array = reinterpret(asset.format, decoded_bytes)
# Allocate the final output array and convert byte order while copying.
out_array = [ltoh(x) for x in reinterpreted_array]
return out_array return out_array
end end
@ -447,23 +510,13 @@ function read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata)
mz = Array{source.mz_format}(undef, meta.mz_asset.encoded_length) mz = Array{source.mz_format}(undef, meta.mz_asset.encoded_length)
intensity = Array{source.intensity_format}(undef, meta.int_asset.encoded_length) intensity = Array{source.intensity_format}(undef, meta.int_asset.encoded_length)
# FIX: Read arrays in the order they appear in the file for efficiency, # Use the new atomic read_at! method for thread-safety
# but ensure we seek to the correct offset for both. read_at!(source.ibd_handle, mz, meta.mz_asset.offset)
if meta.mz_asset.offset < meta.int_asset.offset read_at!(source.ibd_handle, intensity, meta.int_asset.offset)
seek(source.ibd_handle, meta.mz_asset.offset)
read!(source.ibd_handle, mz)
seek(source.ibd_handle, meta.int_asset.offset)
read!(source.ibd_handle, intensity)
else
seek(source.ibd_handle, meta.int_asset.offset)
read!(source.ibd_handle, intensity)
seek(source.ibd_handle, meta.mz_asset.offset)
read!(source.ibd_handle, mz)
end
# imzML data is little-endian. Convert to host byte order. # imzML data is little-endian. Convert to host byte order.
mz .= ltoh.(mz) mz .= ltoh.(mz)
intensity .= ltoh.(intensity) # FIX: Added missing conversion for intensity intensity .= ltoh.(intensity)
validate_spectrum_data(mz, intensity, meta.id) validate_spectrum_data(mz, intensity, meta.id)
@ -733,6 +786,25 @@ function precompute_analytics(msi_data::MSIData)
g_min_mz = minimum(thread_local_min_mz) g_min_mz = minimum(thread_local_min_mz)
g_max_mz = maximum(thread_local_max_mz) g_max_mz = maximum(thread_local_max_mz)
# Only update if we found valid ranges
if isfinite(g_min_mz) && isfinite(g_max_mz) && g_min_mz < g_max_mz
Threads.atomic_xchg!(msi_data.global_min_mz, g_min_mz)
Threads.atomic_xchg!(msi_data.global_max_mz, g_max_mz)
else
# Fallback: calculate from first spectrum
try
if num_spectra > 0
mz, _ = GetSpectrum(msi_data, 1)
if !isempty(mz)
Threads.atomic_xchg!(msi_data.global_min_mz, minimum(mz))
Threads.atomic_xchg!(msi_data.global_max_mz, maximum(mz))
end
end
catch e
@warn "Could not determine global m/z range from first spectrum: $e"
end
end
# Create results # Create results
computed_stats_df = DataFrame( computed_stats_df = DataFrame(
SpectrumID = 1:num_spectra, SpectrumID = 1:num_spectra,
@ -745,10 +817,6 @@ function precompute_analytics(msi_data::MSIData)
Mode = modes Mode = modes
) )
# Set atomic values
Threads.atomic_xchg!(msi_data.global_min_mz, g_min_mz)
Threads.atomic_xchg!(msi_data.global_max_mz, g_max_mz)
# Set other data # Set other data
msi_data.spectrum_stats_df = computed_stats_df msi_data.spectrum_stats_df = computed_stats_df
msi_data.bloom_filters = bloom_filters # Will be created on-demand msi_data.bloom_filters = bloom_filters # Will be created on-demand
@ -1480,30 +1548,3 @@ function get_bloom_filter(data::MSIData, index::Int)::BloomFilter{Int}
return data.bloom_filters[index] return data.bloom_filters[index]
end end
end end
"""
monitor_memory_usage(phase::String)
Utility function to print current memory usage (allocated bytes and RSS)
at different phases of execution.
# Arguments
- `phase`: A string describing the current phase (e.g., "Initialization", "Processing chunk 1").
"""
function monitor_memory_usage(phase::String)
# Use Julia's internal memory tracking
bytes_allocated = Base.gc_bytes()
mb_allocated = bytes_allocated / 1024^2
# Get process memory from system
if Sys.islinux()
proc_status = read("/proc/self/status", String)
vm_rss_match = match(r"VmRSS:\s*(\d+)\s*kB", proc_status)
if vm_rss_match !== nothing
vm_rss_mb = parse(Int, vm_rss_match[1]) / 1024
@printf "[%s] Memory: %.2f MB allocated, %.2f MB RSS\n" phase mb_allocated vm_rss_mb
end
else
@printf "[%s] Memory: %.2f MB allocated\n" phase mb_allocated
end
end

69
src/MSI_src.jl
View File

@ -7,7 +7,6 @@ export OpenMSIData,
GetSpectrum, GetSpectrum,
IterateSpectra, IterateSpectra,
ImportMzmlFile, ImportMzmlFile,
plot_slices,
plot_slice, plot_slice,
get_total_spectrum, get_total_spectrum,
get_average_spectrum, get_average_spectrum,
@ -16,31 +15,34 @@ export OpenMSIData,
generate_colorbar_image, generate_colorbar_image,
process_image_pipeline, process_image_pipeline,
load_and_prepare_mask, load_and_prepare_mask,
set_global_mz_range! set_global_mz_range!,
get_global_mz_range,
MSIData,
_iterate_spectra_fast,
validate_spectrum
# Export the public Preprocessing API # Export the public preprocessing & precalculations API
export FeatureMatrix, export run_preprocessing_analysis,
run_preprocessing_pipeline, main_precalculation,
qc_is_empty, FeatureMatrix,
qc_is_regular, Calibration,
transform_intensity, Smoothing,
smooth_spectrum, BaselineCorrection,
snip_baseline, Normalization,
tic_normalize, PeakPicking,
pqn_normalize, PeakBinningParams,
median_normalize, get_masked_spectrum_indices,
detect_peaks_profile, detect_peaks_profile,
detect_peaks_wavelet,
detect_peaks_centroid, detect_peaks_centroid,
align_peaks_lowess, smooth_spectrum,
find_calibration_peaks, apply_baseline_correction,
apply_normalization,
bin_peaks, bin_peaks,
plot_stage_spectrum, PeakSelection,
calculate_ppm_error, PeakAlignment,
calculate_resolution_fwhm, find_calibration_peaks,
analyze_mass_accuracy, align_peaks_lowess,
generate_qc_report, MutableSpectrum
get_common_calibration_standards
# Include all source files directly into the main module # Include all source files directly into the main module
include("BloomFilters.jl") include("BloomFilters.jl")
@ -52,6 +54,9 @@ include("imzML.jl")
include("MzmlConverter.jl") include("MzmlConverter.jl")
include("Preprocessing.jl") include("Preprocessing.jl")
include("ImageProcessing.jl") include("ImageProcessing.jl")
include("Precalculations.jl")
using Setfield # For immutable struct updates
# --- Main Entry Point --- # # --- Main Entry Point --- #
@ -63,14 +68,28 @@ Opens a .mzML or .imzML file and prepares it for data access.
This is the main entry point for the new data access API. This is the main entry point for the new data access API.
""" """
function OpenMSIData(filepath::String; cache_size=300) function OpenMSIData(filepath::String; cache_size=300, spectrum_type_map::Union{Dict{Int, Symbol}, Nothing}=nothing)
local msi_data
if endswith(lowercase(filepath), ".mzml") if endswith(lowercase(filepath), ".mzml")
return load_mzml_lazy(filepath, cache_size=cache_size) msi_data = load_mzml_lazy(filepath, cache_size=cache_size)
elseif endswith(lowercase(filepath), ".imzml") elseif endswith(lowercase(filepath), ".imzml")
return load_imzml_lazy(filepath, cache_size=cache_size) msi_data = load_imzml_lazy(filepath, cache_size=cache_size)
else else
error("Unsupported file type: $filepath. Please provide a .mzML or .imzML file.") error("Unsupported file type: $filepath. Please provide a .mzML or .imzML file.")
end end
# Apply spectrum type map if provided
if spectrum_type_map !== nothing
for (idx, type_symbol) in spectrum_type_map
if 1 <= idx <= length(msi_data.spectra_metadata)
msi_data.spectra_metadata[idx] = @set msi_data.spectra_metadata[idx].type = type_symbol
else
@warn "Spectrum index $idx out of bounds for spectrum_type_map. Skipping."
end
end
end
return msi_data
end end
end # module MSI_src end # module MSI_src

54
src/MzmlConverter.jl
View File

@ -601,7 +601,7 @@ function ConvertMzmlToImzml(source_file::String, target_ibd_file::String, timing
close(msi_data) close(msi_data)
end end
return binary_meta_vec, coords_vec, (width, height), pixel_modes, ibd_uuid return binary_meta_vec, coords_vec, (width, height), pixel_modes, ibd_uuid, msi_data.instrument_metadata
end end
""" """
@ -621,7 +621,7 @@ experiment and data format.
# Returns # Returns
- `true` on success, `false` on failure. - `true` on success, `false` on failure.
""" """
function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, coords::Vector{Tuple{Int, Int}}, dims::Tuple{Int, Int}, modes::Vector{SpectrumMode}, ibd_uuid::UUID) function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, coords::Vector{Tuple{Int, Int}}, dims::Tuple{Int, Int}, modes::Vector{SpectrumMode}, ibd_uuid::UUID, instrument_meta::InstrumentMetadata)
ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") ibd_file = replace(target_file, r"\.imzML$"i => ".ibd")
if isempty(binary_meta) if isempty(binary_meta)
@ -652,6 +652,15 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
<cvParam cvRef="IMS" accession="IMS:1000080" name="mass spectrum"/> <cvParam cvRef="IMS" accession="IMS:1000080" name="mass spectrum"/>
<cvParam cvRef="IMS" accession="IMS:1000031" name="processed"/> <cvParam cvRef="IMS" accession="IMS:1000031" name="processed"/>
<cvParam cvRef="IMS" accession="IMS:1000081" name="ibd uuid" value="$(ibd_uuid)"/> <cvParam cvRef="IMS" accession="IMS:1000081" name="ibd uuid" value="$(ibd_uuid)"/>
$(
if instrument_meta.polarity == :positive
""" <cvParam cvRef="MS" accession="MS:1000130" name="positive scan"/>"""
elseif instrument_meta.polarity == :negative
""" <cvParam cvRef="MS" accession="MS:1000129" name="negative scan"/>"""
else
""
end
)
</fileContent> </fileContent>
</fileDescription> </fileDescription>
""") """)
@ -693,12 +702,35 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
""") """)
write(imzml_stream, """ <instrumentConfigurationList count="1"> write(imzml_stream, """ <instrumentConfigurationList count="1">
<instrumentConfiguration id="instrument1"> <instrumentConfiguration id="instrument1">
$(
if instrument_meta.instrument_model != "Not Available"
""" <cvParam cvRef="MS" accession="MS:1000031" name="instrument model" value="$(instrument_meta.instrument_model)"/>"""
else
""
end
)
$(
if instrument_meta.resolution !== nothing
""" <cvParam cvRef="MS" accession="MS:1001496" name="mass resolving power" value="$(instrument_meta.resolution)"/>"""
else
""
end
)
<componentList count="3"> <componentList count="3">
<source order="1"> <source order="1">
<cvParam cvRef="MS" accession="MS:1000075" name="MALDI source"/> <cvParam cvRef="MS" accession="MS:1000075" name="MALDI source"/>
</source> </source>
<analyzer order="2"> <analyzer order="2">
<cvParam cvRef="MS" accession="MS:1000084" name="time-of-flight"/> <cvParam cvRef="MS" accession="MS:1000084" name="time-of-flight"/>
$(
if instrument_meta.acquisition_mode == :centroid
""" <cvParam cvRef="MS" accession="MS:1000127" name="centroid spectrum"/>"""
elseif instrument_meta.acquisition_mode == :profile
""" <cvParam cvRef="MS" accession="MS:1000128" name="profile spectrum"/>"""
else
""
end
)
</analyzer> </analyzer>
<detector order="3"> <detector order="3">
<cvParam cvRef="MS" accession="MS:1000253" name="electron multiplier"/> <cvParam cvRef="MS" accession="MS:1000253" name="electron multiplier"/>
@ -708,11 +740,23 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
</instrumentConfiguration> </instrumentConfiguration>
</instrumentConfigurationList> </instrumentConfigurationList>
""") """)
write(imzml_stream, """ <dataProcessingList count="1"> write(imzml_stream, """ <dataProcessingList count="$(1 + (instrument_meta.vendor_preprocessing_steps !== nothing ? length(instrument_meta.vendor_preprocessing_steps) : 0))">
<dataProcessing id="conversionProcessing"> <dataProcessing id="conversionProcessing">
<processingMethod order="1" softwareRef="MSIConverter"> <processingMethod order="1" softwareRef="MSIConverter">
<cvParam cvRef="MS" accession="MS:1000544" name="Conversion to imzML"/> <cvParam cvRef="MS" accession="MS:1000544" name="Conversion to imzML"/>
</processingMethod> </processingMethod>
$(
if instrument_meta.vendor_preprocessing_steps !== nothing
join([
""" <processingMethod order="$(i + 1)" softwareRef="MSIConverter">
<cvParam cvRef="MS" accession="MS:1000589" name="data processing" value="$(step)"/>
</processingMethod>"""
for (i, step) in enumerate(instrument_meta.vendor_preprocessing_steps)
], "\n")
else
""
end
)
</dataProcessing> </dataProcessing>
</dataProcessingList> </dataProcessingList>
""") """)
@ -812,13 +856,13 @@ function ImportMzmlFile(source_file::String, sync_file::String, target_file::Str
println("Step 3: Converting spectra and writing .ibd file...") println("Step 3: Converting spectra and writing .ibd file...")
ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") ibd_file = replace(target_file, r"\.imzML$"i => ".ibd")
binary_meta, coords, (width, height), pixel_modes, ibd_uuid = ConvertMzmlToImzml(source_file, ibd_file, timing_matrix, scans) binary_meta, coords, (width, height), pixel_modes, ibd_uuid, instrument_meta = ConvertMzmlToImzml(source_file, ibd_file, timing_matrix, scans)
# Flip image vertically to match R script output # Flip image vertically to match R script output
flipped_coords = [(x, height - y + 1) for (x, y) in coords] flipped_coords = [(x, height - y + 1) for (x, y) in coords]
println("Step 4: Exporting .imzML metadata file...") println("Step 4: Exporting .imzML metadata file...")
success = ExportImzml(target_file, binary_meta, flipped_coords, (width, height), pixel_modes, ibd_uuid) success = ExportImzml(target_file, binary_meta, flipped_coords, (width, height), pixel_modes, ibd_uuid, instrument_meta)
if success if success
println("Conversion successful: $target_file") println("Conversion successful: $target_file")

1537
src/Precalculations.jl Normal file
View File

File diff suppressed because it is too large Load Diff

1563
src/Preprocessing.jl
View File

File diff suppressed because it is too large Load Diff

446
src/PreprocessingVersioning.jl
View File

@ -1,446 +0,0 @@
# src/PreprocessingVersioning.jl
"""
This module provides a versioning helper functions in our data preprocessing, module
inspired by the functionality of the R package MALDIquant. It includes
functions for quality control, intensity transformation, smoothing, baseline correction,
normalization, peak picking, alignment, and feature matrix generation.
This module is unfinished and may not end up in the release version.
"""
# =============================================================================
# Dependencies
# =============================================================================
using UUIDs, JLD2, CodecBase # For preprocessing versioning
# =============================================================================
# Data Structures
# =============================================================================
"""
VersionedSpectralData
A struct to hold a snapshot of spectral data at a specific point in the
preprocessing workflow. Each transformation creates a new instance of this
struct, forming a history of all operations.
# Fields
- `version_id::String`: A unique identifier for this version of the data.
- `parent_id::String`: The identifier of the version from which this one was derived.
- `spectra::Vector`: The spectral data itself, typically a vector of `(mz, intensity)` tuples.
- `processing_step::String`: A description of the operation that created this version (e.g., "smooth", "baseline").
- `timestamp::DateTime`: The time at which this version was created.
- `parameters::Dict`: A dictionary of the parameters used in the processing step.
"""
struct VersionedSpectralData
version_id::String
parent_id::String
spectra::Vector
processing_step::String
timestamp::DateTime
parameters::Dict
end
"""
MSISession
Manages the state of a preprocessing session, including the history of all
data versions and the current working version.
# Fields
- `session_id::String`: A unique identifier for the entire session.
- `original_data::VersionedSpectralData`: The initial, unprocessed spectral data.
- `history::Vector{VersionedSpectralData}`: A chronological list of all data versions created during the session.
- `current_version::VersionedSpectralData`: The version of the data currently being worked on or displayed.
- `processing_steps::Vector{String}`: A human-readable log of the processing steps applied.
"""
mutable struct MSISession
session_id::String
original_data::VersionedSpectralData
history::Vector{VersionedSpectralData}
current_version::VersionedSpectralData
processing_steps::Vector{String}
end
# =============================================================================
# Session Management
# =============================================================================
"""
initialize_session(raw_spectra::Vector, session_name::String) -> MSISession
Creates a new preprocessing session from raw spectral data.
# Arguments
- `raw_spectra::Vector`: A vector of `(mz, intensity)` tuples representing the initial dataset.
- `session_name::String`: A name for the session.
# Returns
- `MSISession`: A new session object initialized with the provided data.
"""
function initialize_session(raw_spectra::Vector, session_name::String)::MSISession
original = VersionedSpectralData(
string(uuid4()),
"root",
raw_spectra,
"Original Data",
now(),
Dict()
)
return MSISession(session_name, original, [original], original, [])
end
"""
get_processing_history(session::MSISession) -> Vector{String}
Returns a list of the processing steps applied during the session.
"""
get_processing_history(session::MSISession) = [v.processing_step for v in session.history]
"""
undo_last_step!(session::MSISession)
Reverts the session to the state before the last processing step was applied.
This modifies the session in place.
"""
function undo_last_step!(session::MSISession)
if length(session.history) > 1
pop!(session.history)
pop!(session.processing_steps)
session.current_version = last(session.history)
end
return session
end
"""
revert_to_version!(session::MSISession, version_id::String)
Reverts the session to a specific version in its history, discarding all
subsequent changes. This modifies the session in place.
"""
function revert_to_version!(session::MSISession, version_id::String)
target_idx = findfirst(v -> v.version_id == version_id, session.history)
if !isnothing(target_idx)
session.history = session.history[1:target_idx]
session.current_version = session.history[end]
# Also truncate the descriptive processing steps
num_steps_to_keep = max(0, target_idx - 1)
session.processing_steps = session.processing_steps[1:num_steps_to_keep]
end
return session
end
# =============================================================================
# Versioned Preprocessing
# =============================================================================
"""
apply_processing_step(spectra::Vector, step::Symbol, params::Dict) -> Vector
A dispatcher that applies a single, specified preprocessing function to a set of spectra.
This is the core function called by the versioned workflow.
# Arguments
- `spectra::Vector`: The input spectral data.
- `step::Symbol`: The symbol representing the processing step (e.g., `:smooth`, `:baseline`).
- `params::Dict`: A dictionary of parameters for the step.
# Returns
- `Vector`: The processed spectral data.
"""
function apply_processing_step(spectra::Vector, step::Symbol, params::Dict)
processed = deepcopy(spectra)
params_sym = Dict(Symbol(k) => v for (k,v) in params) # Ensure keys are symbols
if step === :transform
for i in eachindex(processed)
mz, y = processed[i]
processed[i] = (mz, transform_intensity(y; params_sym...))
end
elseif step === :smooth
for i in eachindex(processed)
mz, y = processed[i]
processed[i] = (mz, smooth_spectrum(y; params_sym...))
end
elseif step === :baseline
for i in eachindex(processed)
mz, y = processed[i]
baseline = snip_baseline(y; params_sym...)
processed[i] = (mz, max.(0.0, y .- baseline))
end
elseif step === :normalize
mode = get(params, :normalize_method, :tic)
if mode === :pqn
matrix = hcat([float.(p[2]) for p in processed]...)
matrix_norm = pqn_normalize(matrix)
for i in eachindex(processed)
mz, _ = processed[i]
processed[i] = (mz, view(matrix_norm, :, i))
end
else # :tic or :median
norm_func = (mode === :tic) ? tic_normalize : median_normalize
for i in eachindex(processed)
mz, y = processed[i]
processed[i] = (mz, norm_func(y))
end
end
elseif step === :peaks
peak_results = Vector{Tuple{Vector{Float64},Vector{Float64}}}(undef, length(processed))
for (i, (mz, y)) in enumerate(processed)
pk_mz, pk_int = detect_peaks_profile(mz, y; params_sym...)
peak_results[i] = (pk_mz, pk_int)
end
return peak_results
elseif step === :peaks_wavelet
peak_results = Vector{Tuple{Vector{Float64},Vector{Float64}}}(undef, length(processed))
for (i, (mz, y)) in enumerate(processed)
pk_mz, pk_int = detect_peaks_wavelet(mz, y; params_sym...)
peak_results[i] = (pk_mz, pk_int)
end
return peak_results
elseif step === :calibrate
return calibrate_spectra(processed; params_sym...)
else
@warn "Unsupported processing step: $step"
end
return processed
end
"""
run_versioned_preprocessing(session::MSISession, step::Symbol, params::Dict) -> VersionedSpectralData
Creates a new version of spectral data by applying a single processing step to the
current version in the session.
# Arguments
- `session::MSISession`: The current processing session.
- `step::Symbol`: The processing step to apply.
- `params::Dict`: Parameters for the processing step.
# Returns
- `VersionedSpectralData`: A new data version with the transformation applied.
"""
function run_versioned_preprocessing(session::MSISession, step::Symbol, params::Dict)
# Create a new version based on the current one
parent_version = session.current_version
new_version_id = string(uuid4())
# Apply the processing step
new_spectra = apply_processing_step(parent_version.spectra, step, params)
# Create the new versioned data object
new_version = VersionedSpectralData(
new_version_id,
parent_version.version_id,
new_spectra,
string(step),
now(),
params
)
return new_version
end
"""
apply_processing_with_version!(session::MSISession, step::Symbol, ui_params::Dict)
A high-level function to apply a processing step, create a new version, and
update the session state in place.
# Arguments
- `session::MSISession`: The session to modify.
- `step::Symbol`: The processing step to apply.
- `ui_params::Dict`: A dictionary of parameters, typically from a UI.
"""
function apply_processing_with_version!(session::MSISession, step::Symbol, ui_params::Dict)
# In a real app, you would validate and convert UI params here.
processing_params = ui_params
# Create the new version
new_version = run_versioned_preprocessing(session, step, processing_params)
# Update the session
push!(session.history, new_version)
session.current_version = new_version
# Describe the step for the history log
step_description = "$(string(step)) with params: " * join(["$k=$v" for (k,v) in processing_params], ", ")
push!(session.processing_steps, step_description)
return session
end
# =============================================================================
# Data Serialization & Export
# =============================================================================
"""
save_spectral_version(data::VersionedSpectralData, filepath::String)
Saves a `VersionedSpectralData` object to a file using the JLD2 format.
"""
function save_spectral_version(data::VersionedSpectralData, filepath::String)
JLD2.save_object(filepath, data)
end
"""
load_spectral_version(filepath::String) -> VersionedSpectralData
Loads a `VersionedSpectralData` object from a JLD2 file.
"""
function load_spectral_version(filepath::String)::VersionedSpectralData
return JLD2.load_object(filepath)
end
"""
_encode_base64(data::AbstractVector{<:Real}) -> String
Helper function to convert a numeric vector into a Base64 encoded string.
"""
function _encode_base64(data::AbstractVector{T}) where T <: Real
bytes = reinterpret(UInt8, data)
return base64encode(bytes)
end
"""
export_to_mzml(session::MSISession, filepath::String)
Exports the current version of the spectral data in a session to a standard
.mzML file.
# Arguments
- `session::MSISession`: The current session.
- `filepath::String`: The path for the output .mzML file.
"""
function export_to_mzml(session::MSISession, filepath::String)
spectra_to_export = session.current_version.spectra
open(filepath, "w") do f
# XML Header
write(f, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
write(f, "<mzML xmlns=\"http://psi.hupo.org/ms/mzml\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://psi.hupo.org/ms/mzml http://psidev.info/files/ms/mzML/xsd/mzML1.1.0.xsd\" version=\"1.1\">\n")
# CV List
write(f, " <cvList count=\"2\">\n")
write(f, " <cv id=\"MS\" fullName=\"Proteomics Standards Initiative Mass Spectrometry Ontology\" version=\"4.1.123\" URI=\"https://raw.githubusercontent.com/HUPO-PSI/psi-ms-CV/master/psi-ms.obo\"/>\n")
write(f, " <cv id=\"UO\" fullName=\"Unit Ontology\" version=\"07:03:2022\" URI=\"https://raw.githubusercontent.com/bio-ontology-research-group/unit-ontology/master/unit.obo\"/>\n")
write(f, " </cvList>\n")
# File Description
write(f, " <fileDescription>\n <fileContent>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000579\" name=\"MS1 spectrum\"/>\n")
write(f, " </fileContent>\n </fileDescription>\n")
# Run and Spectrum List
write(f, " <run id=\"run1\" defaultInstrumentConfigurationRef=\"instrument1\">\n")
write(f, " <spectrumList count=\"$(length(spectra_to_export))\" defaultDataProcessingRef=\"dp1\">\n")
for (i, (mz, intensity)) in enumerate(spectra_to_export)
# Ensure data is in the correct format for encoding
mz_64 = convert(Vector{Float64}, mz)
int_32 = convert(Vector{Float32}, intensity)
# Base64 encode the binary data
mz_b64 = _encode_base64(mz_64)
int_b64 = _encode_base64(int_32)
write(f, " <spectrum index=\"$(i-1)\" id=\"scan=$(i)\" defaultArrayLength=\"$(length(mz))\">\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000511\" name=\"ms level\" value=\"1\"/>\n")
# Assuming profile mode for processed data, could be made dynamic
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000128\" name=\"profile spectrum\"/>\n")
write(f, " <binaryDataArrayList count=\"2\">\n")
# m/z array
write(f, " <binaryDataArray encodedLength=\"$(length(mz_b64))\">\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000514\" name=\"m/z array\" unitCvRef=\"MS\" unitAccession=\"MS:1000040\" unitName=\"m/z\"/>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000523\" name=\"64-bit float\"/>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000576\" name=\"no compression\"/>\n")
write(f, " <binary>$(mz_b64)</binary>\n")
write(f, " </binaryDataArray>\n")
# Intensity array
write(f, " <binaryDataArray encodedLength=\"$(length(int_b64))\">\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000515\" name=\"intensity array\" unitCvRef=\"MS\" unitAccession=\"MS:1000131\" unitName=\"number of detector counts\"/>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000521\" name=\"32-bit float\"/>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000576\" name=\"no compression\"/>\n")
write(f, " <binary>$(int_b64)</binary>\n")
write(f, " </binaryDataArray>\n")
write(f, " </binaryDataArrayList>\n")
write(f, " </spectrum>\n")
end
write(f, " </spectrumList>\n")
write(f, " </run>\n")
# Dummy instrument and data processing info
write(f, " <instrumentConfigurationList count=\"1\">\n")
write(f, " <instrumentConfiguration id=\"instrument1\"/>\n")
write(f, " </instrumentConfigurationList>\n")
write(f, " <dataProcessingList count=\"1\">\n")
write(f, " <dataProcessing id=\"dp1\"/>\n")
write(f, " </dataProcessingList>\n")
write(f, "</mzML>\n")
end
println("Successfully exported current data to $filepath")
end
# =============================================================================
# Stubs for UI Integration
# =============================================================================
"""
validate_ui_parameters(step::Symbol, ui_params::Dict) -> Tuple{Bool, String}
(Stub) Validates parameters from a UI before they are used in a processing step.
"""
function validate_ui_parameters(step::Symbol, ui_params::Dict)::Tuple{Bool, String}
# In a real implementation, this would check types, ranges, etc.
# For example, for :smooth, ensure 'sg_window' is an odd integer.
println("Validating parameters for step: $step")
return (true, "Parameters are valid.")
end
"""
run_processing_with_progress(session, step, params, progress_callback)
(Stub) A wrapper for running a processing step that includes a progress reporting callback.
"""
function run_processing_with_progress(session, step, params, progress_callback)
progress_callback(0.0, "Starting $step...")
# This is a simplified example. Real implementation would need to
# hook into the loops inside apply_processing_step.
new_version = run_versioned_preprocessing(session, step, params)
progress_callback(1.0, "Finished $step.")
return new_version
end
"""
safe_processing_application!(session::MSISession, step::Symbol, params::Dict)
(Stub) A safe wrapper to apply a processing step that includes error handling and rollback.
"""
function safe_processing_application!(session::MSISession, step::Symbol, params::Dict)
num_history = length(session.history)
try
println("Safely applying step: $step")
apply_processing_with_version!(session, step, params)
catch e
@error "Processing step '$step' failed!" exception=(e, catch_backtrace())
# Roll back to the previous state
while length(session.history) > num_history
undo_last_step!(session)
end
println("Session has been rolled back to the previous state.")
return session # Return the rolled-back session
end
return session # Return the updated session
end

320
src/imzML.jl
View File

@ -17,6 +17,134 @@ Core Functions:
# #
# ============================================================================ # ============================================================================
function parse_imzml_header(stream::IO)
# Initialize all return values and temporary variables
instrument_meta = InstrumentMetadata()
param_groups = Dict{String, SpecDim}()
img_dims = zeros(Int32, 3)
vendor_preprocessing_steps = String[] # Initialize vendor_preprocessing_steps
# Temp vars for parsing
resolution = instrument_meta.resolution
instrument_model = instrument_meta.instrument_model
mass_accuracy_ppm = instrument_meta.mass_accuracy_ppm
polarity = instrument_meta.polarity
calibration_status = instrument_meta.calibration_status
laser_settings = Dict{String, Any}()
try
seekstart(stream)
in_scan_settings = false
dim_axes_found = 0
found_spectrum_count = false
# This loop performs a single pass to get most metadata
while !eof(stream)
line = readline(stream)
# State-machine like logic to enter/exit sections
if occursin("<scanSettings", line)
in_scan_settings = true
elseif occursin("</scanSettings>", line)
in_scan_settings = false
end
# --- Parse Scan Settings for Image Dimensions ---
if in_scan_settings && occursin("<cvParam", line) && dim_axes_found < 2
accession_match = get_attribute(line, "accession")
if accession_match !== nothing
accession = accession_match.captures[1]
if accession == "IMS:1000042" || accession == "IMS:1000043" # max dimension x or y
value_match = get_attribute(line, "value")
if value_match !== nothing
axis_idx = (accession == "IMS:1000042") ? 1 : 2
img_dims[axis_idx] = parse(Int32, value_match.captures[1])
dim_axes_found += 1
end
end
end
end
# --- Parse Spectrum List for total count ---
if occursin("<spectrumList", line)
count_match = get_attribute(line, "count")
if count_match !== nothing
img_dims[3] = parse(Int32, count_match.captures[1])
found_spectrum_count = true
end
end
# --- Parse general CV params for InstrumentMetadata ---
if occursin("<cvParam", line)
accession_match = get_attribute(line, "accession")
value_match = get_attribute(line, "value")
name_match = get_attribute(line, "name") # For laser attributes and vendor_preprocessing names
if accession_match !== nothing
acc = accession_match.captures[1]
val = (value_match !== nothing) ? value_match.captures[1] : ""
cv_name = (name_match !== nothing) ? name_match.captures[1] : "" # Use cv_name to avoid conflict
if acc == "MS:1000031" # instrument model
instrument_model = val
elseif acc == "MS:1001496" # mass resolving power (more specific)
resolution = tryparse(Float64, val)
elseif acc == "MS:1000011" && resolution === nothing # resolution (less specific)
resolution = tryparse(Float64, val)
elseif acc == "MS:1000016" # mass accuracy (ppm)
mass_accuracy_ppm = tryparse(Float64, val)
elseif acc == "MS:1000130" # positive scan
polarity = :positive
elseif acc == "MS:1000129" # negative scan
polarity = :negative
elseif acc == "MS:1000592" # external calibration
calibration_status = :external
elseif acc == "MS:1000593" # internal calibration
calibration_status = :internal
elseif acc == "MS:1000747" && calibration_status == :uncalibrated # instrument specific calibration
calibration_status = :internal # Assume as a form of internal calibration
elseif acc == "MS:1000867" # laser wavelength
laser_settings["wavelength_nm"] = tryparse(Float64, val)
elseif acc == "MS:1000868" # laser fluence
laser_settings["fluence"] = tryparse(Float64, val)
elseif acc == "MS:1000869" # laser repetition rate
laser_settings["repetition_rate_hz"] = tryparse(Float64, val)
# NEW: Vendor Preprocessing terms
elseif acc == "MS:1000579" # baseline correction
push!(vendor_preprocessing_steps, "Baseline Correction")
elseif acc == "MS:1000580" # smoothing
push!(vendor_preprocessing_steps, "Smoothing")
elseif acc == "MS:1000578" # data transformation (e.g., centroiding)
push!(vendor_preprocessing_steps, "Data Transformation: $(cv_name)") # Use cv_name here
elseif acc == "MS:1000800" # deisotoping
push!(vendor_preprocessing_steps, "Deisotoping")
end
end
end
if found_spectrum_count && dim_axes_found == 2
break
end
end
catch e
@warn "Could not fully parse imzML header in single pass. Error: $e"
end
# The axes_config_img logic is complex to merge; we run it as a second, small pass.
seekstart(stream)
param_groups = axes_config_img(stream)
final_instrument_meta = InstrumentMetadata(
resolution, :unknown, :unknown, calibration_status,
instrument_model, mass_accuracy_ppm,
isempty(laser_settings) ? nothing : laser_settings,
polarity,
isempty(vendor_preprocessing_steps) ? nothing : vendor_preprocessing_steps # NEW field
)
return (final_instrument_meta, param_groups, img_dims)
end
""" """
axes_config_img(stream) axes_config_img(stream)
@ -44,40 +172,6 @@ function axes_config_img(stream::IO)
return param_groups return param_groups
end end
"""
get_img_dimensions(stream)
Reads the maximum X and Y dimensions and total spectrum count from the metadata.
"""
function get_img_dimensions(stream::IO)
find_tag(stream, r"^\s*<(scanSettings )")
n = 2
dim = zeros(Int32, 3)
while n > 0 && !eof(stream)
currLine = readline(stream)
if occursin("<cvParam", currLine)
accession_match = get_attribute(currLine, "accession")
if accession_match !== nothing
accession = accession_match.captures[1]
if accession == "IMS:1000042" || accession == "IMS:1000043"
value_match = get_attribute(currLine, "value")
if value_match !== nothing
axis_idx = (accession == "IMS:1000042") ? 1 : 2
dim[axis_idx] = parse(Int32, value_match.captures[1])
n -= 1
end
end
end
end
end
count_tag = find_tag(stream, r"^\s*<spectrumList(.+)")
count_match = get_attribute(count_tag.captures[1], "count")
dim[3] = parse(Int32, count_match.captures[1])
return dim
end
""" """
get_spectrum_tag_offset(stream) get_spectrum_tag_offset(stream)
@ -179,13 +273,9 @@ function determine_parser(stream::IO, mz_is_compressed::Bool, int_is_compressed:
return :neofx return :neofx
end end
if mz_is_compressed || int_is_compressed || has_external_data_markers # If not neofx, it must be compressed (as uncompressed path is no longer supported)
return :compressed return :compressed
end end
return :uncompressed
end
function load_imzml_lazy(file_path::String; cache_size::Int=100) function load_imzml_lazy(file_path::String; cache_size::Int=100)
println("DEBUG: Checking for .imzML file at $file_path") println("DEBUG: Checking for .imzML file at $file_path")
if !isfile(file_path) if !isfile(file_path)
@ -203,10 +293,21 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
ts_hIbd = ThreadSafeFileHandle(ibd_path) ts_hIbd = ThreadSafeFileHandle(ibd_path)
try try
println("DEBUG: Configuring axes...") # --- NEW: Parse all header information in a more efficient single pass ---
param_groups = axes_config_img(stream) println("DEBUG: Parsing imzML header...")
println("DEBUG: Getting image dimensions...") (instrument_meta, param_groups, imgDim) = parse_imzml_header(stream)
imgDim = get_img_dimensions(stream) # The header parser will have reset the stream for the next step (spectrum parsing)
println("--- Extracted Instrument Metadata ---")
println("Resolution: ", instrument_meta.resolution)
println("Acquisition Mode (pre-check): ", instrument_meta.acquisition_mode)
println("Calibration Status: ", instrument_meta.calibration_status)
println("Instrument Model: ", instrument_meta.instrument_model)
println("Mass Accuracy (ppm): ", instrument_meta.mass_accuracy_ppm)
println("Laser Settings: ", instrument_meta.laser_settings)
println("Polarity: ", instrument_meta.polarity)
println("------------------------------------")
width, height, num_spectra = imgDim width, height, num_spectra = imgDim
println("DEBUG: Image dimensions: $(width)x$(height), $num_spectra spectra.") println("DEBUG: Image dimensions: $(width)x$(height), $num_spectra spectra.")
@ -251,19 +352,6 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
spectra_metadata = parse_neofx(stream, ts_hIbd, param_groups, width, height, num_spectra, spectra_metadata = parse_neofx(stream, ts_hIbd, param_groups, width, height, num_spectra,
default_mz_format, default_intensity_format, default_mz_format, default_intensity_format,
mz_is_compressed, int_is_compressed, global_mode) mz_is_compressed, int_is_compressed, global_mode)
#=
elseif parser_type == :compressed
println("DEBUG: Using compressed parser.")
spectra_metadata = parse_compressed(stream, hIbd, param_groups, width, height, num_spectra,
default_mz_format, default_intensity_format,
mz_is_compressed, int_is_compressed, global_mode)
else # :uncompressed
println("DEBUG: Using uncompressed parser.")
spectra_metadata = parse_uncompressed(stream, hIbd, param_groups, width, height, num_spectra,
default_mz_format, default_intensity_format,
mz_is_compressed, int_is_compressed, global_mode)
end
=#
else else
println("DEBUG: Using compressed parser.") println("DEBUG: Using compressed parser.")
spectra_metadata = parse_compressed(stream, ts_hIbd, param_groups, width, height, num_spectra, spectra_metadata = parse_compressed(stream, ts_hIbd, param_groups, width, height, num_spectra,
@ -286,9 +374,30 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
end end
println("DEBUG: Coordinate map built.") println("DEBUG: Coordinate map built.")
# --- NEW: Update acquisition mode based on spectrum parsing ---
acq_mode_symbol = if global_mode == CENTROID
:centroid
elseif global_mode == PROFILE
:profile
else
:unknown
end
final_instrument_meta = InstrumentMetadata(
instrument_meta.resolution,
acq_mode_symbol, # Update with parsed mode
instrument_meta.mz_axis_type,
instrument_meta.calibration_status,
instrument_meta.instrument_model,
instrument_meta.mass_accuracy_ppm,
instrument_meta.laser_settings,
instrument_meta.polarity,
instrument_meta.vendor_preprocessing_steps # Add this new field
)
source = ImzMLSource(ts_hIbd, default_mz_format, default_intensity_format) source = ImzMLSource(ts_hIbd, default_mz_format, default_intensity_format)
println("DEBUG: Creating MSIData object.") println("DEBUG: Creating MSIData object.")
msi_data = MSIData(source, spectra_metadata, (width, height), coordinate_map, cache_size) msi_data = MSIData(source, spectra_metadata, final_instrument_meta, (width, height), coordinate_map, cache_size)
# Close the XML stream as it's no longer needed # Close the XML stream as it's no longer needed
close(stream) close(stream)
@ -302,99 +411,6 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
end end
end end
#=
function parse_uncompressed(stream::IO, hIbd::IO, param_groups::Dict{String, SpecDim},
width::Int32, height::Int32, num_spectra::Int32,
mz_format::DataType, intensity_format::DataType,
mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode)
println("DEBUG: Learning file structure from first spectrum...")
start_of_spectra_xml = position(stream)
# Skip UUID at beginning of IBD file (from converter)
seek(hIbd, 16)
current_ibd_offset = position(hIbd)
attr = get_spectrum_attributes(stream, hIbd)
seek(stream, start_of_spectra_xml)
println("DEBUG: Initial IBD offset (after UUID): $current_ibd_offset")
spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra)
mz_is_first = attr[3] == 3
for k in 1:num_spectra
spectrum_start_pos = position(stream)
# Find X coordinate
line = readuntil(stream, "IMS:1000050")
if eof(stream)
throw(FileFormatError("Unexpected EOF while looking for X coordinate"))
end
val_tag_x = find_tag(stream, r"value=\"(\d+)\"")
x = parse(Int32, val_tag_x.captures[1])
# Find Y coordinate
line = readuntil(stream, "IMS:1000051")
val_tag_y = find_tag(stream, r"value=\"(\d+)\"")
y = parse(Int32, val_tag_y.captures[1])
# Find array length - look for external array length
line = readuntil(stream, "IMS:1000103")
val_tag_len = find_tag(stream, r"value=\"(\d+)\"")
nPoints = parse(Int32, val_tag_len.captures[1])
# CRITICAL FIX: Use the actual formats from the XML, not hardcoded values
# The converter writes Float64 for mz, Float32 for intensity
mz_len_bytes = nPoints * sizeof(Float64) # Converter uses Float64 for mz
int_len_bytes = nPoints * sizeof(Float32) # Converter uses Float32 for intensity
local mz_offset, int_offset
if mz_is_first
mz_offset = current_ibd_offset
int_offset = mz_offset + mz_len_bytes
else
int_offset = current_ibd_offset
mz_offset = int_offset + int_len_bytes
end
# Mode detection
current_pos = position(stream)
seek(stream, spectrum_start_pos)
spectrum_buffer = IOBuffer()
line = ""
while !eof(stream)
line = readline(stream)
write(spectrum_buffer, line)
if occursin("</spectrum>", line)
break
end
end
spectrum_xml = String(take!(spectrum_buffer))
spectrum_mode = global_mode
if occursin("MS:1000127", spectrum_xml)
spectrum_mode = CENTROID
elseif occursin("MS:1000128", spectrum_xml)
spectrum_mode = PROFILE
end
seek(stream, current_pos)
# CRITICAL FIX: Use correct data types that match the converter output
mz_asset = SpectrumAsset(Float64, mz_is_compressed, mz_offset, nPoints, :mz)
int_asset = SpectrumAsset(Float32, int_is_compressed, int_offset, nPoints, :intensity)
spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset)
current_ibd_offset += mz_len_bytes + int_len_bytes
# Skip to next spectrum
line = readuntil(stream, "</spectrum>")
end
return spectra_metadata
end
=#
function parse_compressed(stream::IO, hIbd::Union{IO, ThreadSafeFileHandle}, param_groups::Dict{String, SpecDim}, function parse_compressed(stream::IO, hIbd::Union{IO, ThreadSafeFileHandle}, param_groups::Dict{String, SpecDim},
width::Int32, height::Int32, num_spectra::Int32, width::Int32, height::Int32, num_spectra::Int32,
default_mz_format::DataType, default_intensity_format::DataType, default_mz_format::DataType, default_intensity_format::DataType,
@ -532,7 +548,7 @@ function parse_compressed(stream::IO, hIbd::Union{IO, ThreadSafeFileHandle}, par
int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity) int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity)
end end
spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset) spectra_metadata[k] = SpectrumMetadata(x, y, "", :sample, spectrum_mode, mz_asset, int_asset)
end end
return spectra_metadata return spectra_metadata
@ -675,7 +691,7 @@ function parse_neofx(stream::IO, hIbd::Union{IO, ThreadSafeFileHandle}, param_gr
int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity) int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity)
end end
spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset) spectra_metadata[k] = SpectrumMetadata(x, y, "", :sample, spectrum_mode, mz_asset, int_asset)
end end
return spectra_metadata return spectra_metadata

148
src/mzML.jl
View File

@ -19,6 +19,107 @@ const DATA_FORMAT_ACCESSIONS = Dict{String, DataType}(
"MS:1000523" => Float64 "MS:1000523" => Float64
) )
function parse_instrument_metadata_mzml(stream::IO)
# Initialize with default values from the InstrumentMetadata constructor
instrument_meta = InstrumentMetadata()
# Create temporary variables to hold parsed values
resolution = instrument_meta.resolution
instrument_model = instrument_meta.instrument_model
mass_accuracy_ppm = instrument_meta.mass_accuracy_ppm
polarity = instrument_meta.polarity
calibration_status = instrument_meta.calibration_status
laser_settings = Dict{String, Any}() # Use Any for heterogeneous values
vendor_preprocessing_steps = String[] # Initialize vendor_preprocessing_steps
try
# Reset stream and read a sufficiently large header block to find metadata.
seekstart(stream)
header_block = ""
header_read_limit = 50000 # Read up to 50KB of header
bytes_read = 0
while !eof(stream) && bytes_read < header_read_limit
line_pos = position(stream)
line = readline(stream)
bytes_read += (position(stream) - line_pos)
# Stop at the start of the main data section
if occursin("<run>", line) || occursin("<spectrumList>", line)
break
end
header_block *= line * "\n"
end
header_io = IOBuffer(header_block)
while !eof(header_io)
line = readline(header_io)
if occursin("<cvParam", line)
accession_match = get_attribute(line, "accession")
value_match = get_attribute(line, "value")
name_match = get_attribute(line, "name") # For laser attributes and vendor_preprocessing names
if accession_match !== nothing
acc = accession_match.captures[1]
val = (value_match !== nothing) ? value_match.captures[1] : ""
name = (name_match !== nothing) ? name_match.captures[1] : "" # NEW: Get name for logging
if acc == "MS:1000031" # instrument model
instrument_model = val
elseif acc == "MS:1001496" # mass resolving power (more specific)
resolution = tryparse(Float64, val)
elseif acc == "MS:1000011" && resolution === nothing # resolution (less specific)
resolution = tryparse(Float64, val)
elseif acc == "MS:1000016" # mass accuracy (ppm)
mass_accuracy_ppm = tryparse(Float64, val)
elseif acc == "MS:1000130" # positive scan
polarity = :positive
elseif acc == "MS:1000129" # negative scan
polarity = :negative
elseif acc == "MS:1000592" # external calibration
calibration_status = :external
elseif acc == "MS:1000593" # internal calibration
calibration_status = :internal
elseif acc == "MS:1000747" && calibration_status == :uncalibrated # instrument specific calibration
calibration_status = :internal # Assume as a form of internal calibration
elseif acc == "MS:1000867" # laser wavelength
laser_settings["wavelength_nm"] = tryparse(Float64, val)
elseif acc == "MS:1000868" # laser fluence
laser_settings["fluence"] = tryparse(Float64, val)
elseif acc == "MS:1000869" # laser repetition rate
laser_settings["repetition_rate_hz"] = tryparse(Float64, val)
# NEW: Vendor Preprocessing terms
elseif acc == "MS:1000579" # baseline correction
push!(vendor_preprocessing_steps, "Baseline Correction")
elseif acc == "MS:1000580" # smoothing
push!(vendor_preprocessing_steps, "Smoothing")
elseif acc == "MS:1000578" # data transformation (e.g., centroiding)
push!(vendor_preprocessing_steps, "Data Transformation: $(name)")
elseif acc == "MS:1000800" # deisotoping
push!(vendor_preprocessing_steps, "Deisotoping")
end
end
end
end
catch e
@warn "Could not fully parse instrument metadata from mzML header. Using defaults. Error: $e"
end
# Always return a valid object
return InstrumentMetadata(
resolution,
instrument_meta.acquisition_mode, # To be determined by load_mzml_lazy
instrument_meta.mz_axis_type,
calibration_status,
instrument_model,
mass_accuracy_ppm,
isempty(laser_settings) ? nothing : laser_settings,
polarity,
isempty(vendor_preprocessing_steps) ? nothing : vendor_preprocessing_steps # NEW field
)
end
""" """
get_spectrum_asset_metadata(stream::IO) get_spectrum_asset_metadata(stream::IO)
@ -149,7 +250,7 @@ function parse_spectrum_metadata(stream::IO, offset::Int64)
# Create the new unified metadata object # Create the new unified metadata object
# For mzML, x and y coordinates are not applicable, so we use 0. # For mzML, x and y coordinates are not applicable, so we use 0.
return SpectrumMetadata(Int32(0), Int32(0), id, mode, mz_asset, int_asset) return SpectrumMetadata(Int32(0), Int32(0), id, :sample, mode, mz_asset, int_asset)
end end
""" """
@ -231,6 +332,22 @@ function load_mzml_lazy(file_path::String; cache_size::Int=100)
ts_stream = ThreadSafeFileHandle(file_path, "r") ts_stream = ThreadSafeFileHandle(file_path, "r")
try try
# --- NEW: Parse instrument metadata from header ---
println("DEBUG: Parsing instrument metadata from header...")
instrument_meta = parse_instrument_metadata_mzml(ts_stream.handle)
println("--- Extracted Instrument Metadata ---")
println("Resolution: ", instrument_meta.resolution)
println("Acquisition Mode (pre-check): ", instrument_meta.acquisition_mode)
println("Calibration Status: ", instrument_meta.calibration_status)
println("Instrument Model: ", instrument_meta.instrument_model)
println("Mass Accuracy (ppm): ", instrument_meta.mass_accuracy_ppm)
println("Laser Settings: ", instrument_meta.laser_settings)
println("Polarity: ", instrument_meta.polarity)
println("------------------------------------")
seekstart(ts_stream.handle) # Reset stream after header parsing
println("DEBUG: Finding index offset...") println("DEBUG: Finding index offset...")
index_offset = find_index_offset(ts_stream.handle) index_offset = find_index_offset(ts_stream.handle)
println("DEBUG: Seeking to index list at offset $index_offset.") println("DEBUG: Seeking to index list at offset $index_offset.")
@ -270,9 +387,36 @@ function load_mzml_lazy(file_path::String; cache_size::Int=100)
mz_format = first_meta.mz_asset.format mz_format = first_meta.mz_asset.format
intensity_format = first_meta.int_asset.format intensity_format = first_meta.int_asset.format
# --- NEW: Determine overall acquisition mode ---
modes = [meta.mode for meta in spectra_metadata]
num_centroid = count(m -> m == CENTROID, modes)
num_profile = count(m -> m == PROFILE, modes)
acq_mode_symbol = if num_centroid > 0 && num_profile == 0
:centroid
elseif num_profile > 0 && num_centroid == 0
:profile
elseif num_centroid > 0 && num_profile > 0
:mixed
else
:unknown
end
final_instrument_meta = InstrumentMetadata(
instrument_meta.resolution,
acq_mode_symbol, # Update with parsed mode
instrument_meta.mz_axis_type,
instrument_meta.calibration_status,
instrument_meta.instrument_model,
instrument_meta.mass_accuracy_ppm,
instrument_meta.laser_settings,
instrument_meta.polarity,
instrument_meta.vendor_preprocessing_steps # Add this new field
)
source = MzMLSource(ts_stream, mz_format, intensity_format) source = MzMLSource(ts_stream, mz_format, intensity_format)
println("DEBUG: Creating MSIData object.") println("DEBUG: Creating MSIData object.")
return MSIData(source, spectra_metadata, (0, 0), nothing, cache_size) return MSIData(source, spectra_metadata, final_instrument_meta, (0, 0), nothing, cache_size)
catch e catch e
close(ts_stream) # Ensure stream is closed on error close(ts_stream) # Ensure stream is closed on error

222
test/run_descriptive_preprocessing.jl Normal file
View File

@ -0,0 +1,222 @@
using CairoMakie
using Statistics
using Colors
function create_msi_parameter_plot()
# Create the figure with subplots for each preprocessing step
fig = Figure(size=(1600, 1200), fontsize=12)
# Define the preprocessing steps and their parameters
steps = [
("BaselineCorrection", ["iterations: 10", "method: SNIP", "window: 3.0"]),
("Calibration", ["fit_order: 2", "method: internal_standards", "ppm_tolerance: 20.0"]),
("Normalization", ["method: rms"]),
("PeakAlignment", ["max_shift_ppm: 50.0", "method: linear", "tolerance: 20.0 ppm"]),
("PeakBinningParams", ["max_bin_width_ppm: 60.0", "method: adaptive", "min_peak_per_bin: 3"]),
("PeakPicking", ["half_window: 5", "method: centroid", "snr_threshold: 15.0"]),
("PeakSelection", ["max_fwhm_ppm: 95.39", "min_fwhm_ppm: 19.08", "min_snr: 3.0"]),
("Smoothing", ["method: savitzky_golay", "order: 3", "window: 5"])
]
# Create a grid of subplots
g = fig[1, 1] = GridLayout()
# Plot each preprocessing step
for (idx, (step_name, params)) in enumerate(steps)
row, col = fldmod1(idx, 2)
ax = Axis(g[row, col], title=step_name, titlesize=14)
# Generate simulated m/z values and intensities
mz_range = range(100, 1000, length=500)
if step_name == "BaselineCorrection"
# Simulate spectrum with baseline
true_signal = 0.5 .* exp.(-0.001 .* (mz_range .- 400).^2) .+
0.3 .* exp.(-0.002 .* (mz_range .- 600).^2)
baseline = 0.1 .+ 0.05 .* sin.(mz_range ./ 50)
noisy_signal = true_signal .+ baseline .+ 0.02 .* randn(length(mz_range))
corrected = noisy_signal .- baseline
lines!(ax, mz_range, noisy_signal, color=:blue, linewidth=2, label="Raw")
lines!(ax, mz_range, baseline, color=:red, linewidth=2, linestyle=:dash, label="Baseline")
lines!(ax, mz_range, corrected, color=:green, linewidth=2, label="Corrected")
elseif step_name == "Calibration"
# Simulate calibration shift
reference_peaks = [200, 400, 600, 800]
measured_peaks = reference_peaks .+ 2.0 .* randn(length(reference_peaks))
scatter!(ax, reference_peaks, fill(0.5, length(reference_peaks)),
color=:red, markersize=15, label="Reference")
scatter!(ax, measured_peaks, fill(0.3, length(measured_peaks)),
color=:blue, markersize=10, label="Measured")
# Add calibration lines
for i in 1:length(reference_peaks)
lines!(ax, [measured_peaks[i], reference_peaks[i]], [0.3, 0.5],
color=:black, linewidth=1, linestyle=:dash)
end
elseif step_name == "Normalization"
# Simulate normalization effect
spectra = [
0.8 .* exp.(-0.001 .* (mz_range .- 300).^2) .+ 0.2 .* randn(length(mz_range)),
1.2 .* exp.(-0.001 .* (mz_range .- 300).^2) .+ 0.2 .* randn(length(mz_range)),
0.9 .* exp.(-0.001 .* (mz_range .- 300).^2) .+ 0.2 .* randn(length(mz_range))
]
normalized_spectra = [spec ./ std(spec) for spec in spectra]
for (i, spec) in enumerate(spectra)
lines!(ax, mz_range, spec .+ i*0.3, color=RGBA(1, 0, 0, 0.6), linewidth=2,
label=i==1 ? "Before Norm" : "")
end
for (i, spec) in enumerate(normalized_spectra)
lines!(ax, mz_range, spec .+ i*0.3, color=RGBA(0, 0, 1, 0.6), linewidth=2,
label=i==1 ? "After Norm" : "")
end
elseif step_name == "PeakAlignment"
# Simulate peak alignment
base_peaks = [300, 500, 700]
shifts = [-15, 5, 10]
for (i, shift) in enumerate(shifts)
shifted_peaks = base_peaks .+ shift
aligned_peaks = base_peaks
scatter!(ax, shifted_peaks, fill(i, length(shifted_peaks)),
color=:red, markersize=12, label=i==1 ? "Before Align" : "")
scatter!(ax, aligned_peaks, fill(i+0.3, length(aligned_peaks)),
color=:green, markersize=12, label=i==1 ? "After Align" : "")
# Show alignment lines
for j in 1:length(base_peaks)
lines!(ax, [shifted_peaks[j], aligned_peaks[j]], [i, i+0.3],
color=:black, linewidth=1, linestyle=:dash)
end
end
elseif step_name == "PeakBinningParams"
# Simulate peak binning with centroids
raw_peaks_mz = 100:25:900
raw_peaks_intensity = rand(length(raw_peaks_mz))
# Create binned peaks (wider bins)
bin_centers = 150:60:850
bin_intensities = [sum(raw_peaks_intensity[abs.(raw_peaks_mz .- center) .< 30])
for center in bin_centers] .* 0.8
# Profile mode (continuous)
profile_signal = zeros(length(mz_range))
for (mz, int) in zip(raw_peaks_mz, raw_peaks_intensity)
profile_signal .+= int .* exp.(-0.001 .* (mz_range .- mz).^2)
end
lines!(ax, mz_range, profile_signal, color=:blue, linewidth=2, label="Profile")
barplot!(ax, bin_centers, bin_intensities, color=RGBA(1, 0, 0, 0.7),
width=50, label="Binned Centroids")
elseif step_name == "PeakPicking"
# Simulate peak picking from profile to centroids
profile_signal = 0.6 .* exp.(-0.0005 .* (mz_range .- 400).^2) .+
0.4 .* exp.(-0.0008 .* (mz_range .- 650).^2) .+
0.1 .* randn(length(mz_range))
# Simulate picked peaks (centroids)
peak_positions = [380, 405, 640, 660]
peak_intensities = [0.5, 0.6, 0.35, 0.4]
lines!(ax, mz_range, profile_signal, color=:blue, linewidth=2, label="Profile Spectrum")
scatter!(ax, peak_positions, peak_intensities, color=:red, markersize=20,
label="Picked Centroids", strokewidth=2)
elseif step_name == "PeakSelection"
# Simulate peak selection based on criteria
all_peaks_mz = 200:50:800
all_peaks_fwhm = rand(length(all_peaks_mz)) .* 100 .+ 10
all_peaks_snr = rand(length(all_peaks_mz)) .* 10
# Selection criteria
selected = (all_peaks_fwhm .>= 19.08) .& (all_peaks_fwhm .<= 95.39) .& (all_peaks_snr .>= 3.0)
scatter!(ax, all_peaks_mz[.!selected], all_peaks_fwhm[.!selected],
color=:red, markersize=15, label="Rejected")
scatter!(ax, all_peaks_mz[selected], all_peaks_fwhm[selected],
color=:green, markersize=15, label="Selected")
# Add selection criteria lines
hlines!(ax, [19.08, 95.39], color=:black, linestyle=:dash, linewidth=2)
text!(ax, 850, 50; text="FWHM bounds", color=:black, fontsize=10)
elseif step_name == "Smoothing"
# Simulate smoothing effect
true_signal = 0.7 .* exp.(-0.001 .* (mz_range .- 450).^2) .+
0.5 .* exp.(-0.0008 .* (mz_range .- 650).^2)
noisy_signal = true_signal .+ 0.1 .* randn(length(mz_range))
# Simple smoothing simulation
smoothed = similar(noisy_signal)
window = 5
for i in 1:length(noisy_signal)
start_idx = max(1, i - window ÷ 2)
end_idx = min(length(noisy_signal), i + window ÷ 2)
smoothed[i] = mean(noisy_signal[start_idx:end_idx])
end
lines!(ax, mz_range, noisy_signal, color=:red, linewidth=1, label="Noisy")
lines!(ax, mz_range, smoothed, color=:blue, linewidth=2, label="Smoothed")
lines!(ax, mz_range, true_signal, color=:green, linewidth=1, linestyle=:dash, label="True")
end
# Add parameter text using a more reliable approach
param_text = join(params, "\n")
text!(ax, param_text, position=Point2f(0.05, 0.95),
space=:relative, align=(:left, :top), color=:black,
fontsize=10, font=:regular)
# Add legend for selected plots
if idx <= 4
axislegend(ax, position=:rt, framevisible=true, backgroundcolor=RGBA(1,1,1,0.8))
end
# Customize axes
ax.xlabel = "m/z"
ax.ylabel = idx in [1,3,5,7] ? "Intensity" : ""
ax.xgridvisible = false
ax.ygridvisible = false
end
# Add overall title
Label(fig[0, :], "MSI Preprocessing Pipeline Parameters and Simulations",
fontsize=18, font=:bold, padding=(0, 0, 10, 0))
# Add explanation
explanation = """
Simulation of MSI preprocessing parameters showing:
• Blue lines: Profile/continuous spectra
• Red bars/points: Centroid data
• Dashed lines: Reference/true signals
• Green: Processed/corrected data
• Each subplot demonstrates key parameters for the preprocessing step
"""
Label(fig[2, :], explanation, fontsize=12, tellwidth=false, padding=(10, 10, 10, 10))
# Adjust layout
colgap!(g, 20)
rowgap!(g, 20)
fig
end
# Create and display the plot
fig = create_msi_parameter_plot()
# Save the plot
save("msi_preprocessing_parameters.png", fig)
println("Plot saved as 'msi_preprocessing_parameters.png'")
# Display the plot (if in an interactive environment)
fig

157
test/run_precalculation_dict.jl Normal file
View File

@ -0,0 +1,157 @@
# test/run_precalculation_example.jl
using Printf
import Pkg
# --- Load the MSI_src Module ---
Pkg.activate(joinpath(@__DIR__, ".."))
using MSI_src
# ===================================================================
# CONFIG: PLEASE FILL IN YOUR FILE PATHS HERE
# ===================================================================
# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML"
#const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/mzML/Col_1.mzML"
const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.mzML"
# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.imzML"
#const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML"
const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
#const MASK_ROUTE = "/home/pixel/Documents/Cinvestav_2025/JuliaMSI/public/css/masks/Stomach_DHB_uncompressed.png"
const MASK_ROUTE = ""
const reference_peaks = Dict(
# DHB Matrix peaks (should be present)
137.0244 => "DHB_fragment",
155.0349 => "DHB_M+H",
177.0168 => "DHB_M+Na",
# Common lipids in your mass range
496.3398 => "PC_16:0_16:0",
520.3398 => "PC_16:0_18:1",
760.5851 => "PC_16:0_18:1_Na",
# Common contaminants
391.2843 => "PDMS",
413.2662 => "PDMS_Na",
# Add some high mass peaks
842.5092 => "Protein_standard",
1045.532 => "Protein_standard",
290.1747 => "Atropine [M+H]+",
304.1903 => "Scopolamine [M+H]+",
124.0393 => "Tropine [M+H]+",
)
# ===================================================================
# HELPER FUNCTIONS FOR PRINTING
# ===================================================================
function print_header(title::String)
println("\n" * "="^80)
println("$(title)")
println("="^80)
end
function check_data_range(msi_data::MSIData)
println("\n--- Data Range Analysis ---")
min_mz, max_mz = get_global_mz_range(msi_data)
if isfinite(min_mz) && isfinite(max_mz) && min_mz < max_mz
println("Global m/z range: [$(min_mz), $(max_mz)]")
else
println("Global m/z range: Not yet determined or invalid (initial: [$(min_mz), $(max_mz)])")
end
# Check a few spectra to see actual m/z values
println("\nChecking first few spectra for actual m/z values:")
for i in 1:min(3, length(msi_data.spectra_metadata))
try
mz, intensity = GetSpectrum(msi_data, i)
if !isempty(mz)
println("Spectrum $i: m/z range [$(minimum(mz)), $(maximum(mz))], length=$(length(mz))")
# Print first and last few m/z values
if length(mz) > 10
println(" First 5 m/z: $(mz[1:5])")
println(" Last 5 m/z: $(mz[end-4:end])")
end
end
catch e
println("Spectrum $i: Error - $e")
end
end
end
# ===================================================================
# MAIN EXAMPLE RUNNER
# ===================================================================
function run_precalculation_example()
# --- Process mzML file ---
print_header("Processing mzML File: $(basename(TEST_MZML_FILE))")
if !isfile(TEST_MZML_FILE)
println("SKIPPING: mzML file not found at $(TEST_MZML_FILE)")
else
try
msi_data_mzml = @time OpenMSIData(TEST_MZML_FILE)
check_data_range(msi_data_mzml)
analysis_results_mzml = main_precalculation(msi_data_mzml, reference_peaks=reference_peaks)
println("\n" * "*"^80)
println("MZML PREPROCESSING ANALYSIS RESULTS")
println("*"^80)
for (step_name, params) in analysis_results_mzml
println("\n--- $(uppercase(string(step_name))) Parameters ---")
if isempty(params)
println(" No recommended parameters.")
else
for (param_name, param_value) in params
println(" $param_name: $param_value")
end
end
end
close(msi_data_mzml) # Close file handles
catch e
println("ERROR processing mzML file: $e")
showerror(stdout, e, catch_backtrace())
end
end
# --- Process imzML file ---
print_header("Processing imzML File: $(basename(TEST_IMZML_FILE))")
if !isfile(TEST_IMZML_FILE)
println("SKIPPING: imzML file not found at $(TEST_IMZML_FILE)")
else
try
msi_data_imzml = @time OpenMSIData(TEST_IMZML_FILE)
check_data_range(msi_data_imzml)
analysis_results_imzml = main_precalculation(msi_data_imzml, reference_peaks=reference_peaks, mask_path=MASK_ROUTE)
println("\n" * "*"^80)
println("IMZML PREPROCESSING ANALYSIS RESULTS")
println("*"^80)
for (step_name, params) in analysis_results_imzml
println("\n--- $(uppercase(string(step_name))) Parameters ---")
if isempty(params)
println(" No recommended parameters.")
else
for (param_name, param_value) in params
println(" $param_name: $param_value")
end
end
end
close(msi_data_imzml) # Close file handles
catch e
println("ERROR processing imzML file: $e")
showerror(stdout, e, catch_backtrace())
end
end
end
# --- Execute ---
@time run_precalculation_example()

281
test/run_precalculation_example.jl Normal file
View File

@ -0,0 +1,281 @@
# test/run_precalculation_example.jl
using Printf
import Pkg
# --- Load the MSI_src Module ---
Pkg.activate(joinpath(@__DIR__, ".."))
using MSI_src
# ===================================================================
# CONFIG: PLEASE FILL IN YOUR FILE PATHS HERE
# ===================================================================
# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML"
# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/mzML/Col_1.mzML"
const TEST_MZML_FILE = ""
# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.imzML"
#const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML"
const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
#const MASK_ROUTE = "/home/pixel/Documents/Cinvestav_2025/JuliaMSI/public/css/masks/Stomach_DHB_uncompressed.png"
const MASK_ROUTE = ""
#=
const reference_peaks = Dict(
# Common ESI positive mode reference compounds
121.0509 => "Purine",
149.0233 => "HP-921",
322.0481 => "Hexakis(1H,1H,3H-tetrafluoropropoxy)phosphazine",
622.0290 => "Hexakis(2,2-difluoroethoxy)phosphazine",
# Atropine and related compounds in positive mode
290.1747 => "Atropine [M+H]+",
304.1903 => "Scopolamine [M+H]+",
124.0393 => "Tropine [M+H]+",
# Common contaminants and lock masses
391.2843 => "Polydimethylcyclosiloxane [M+H]+",
413.2662 => "Polydimethylcyclosiloxane [M+Na]+",
429.2402 => "Polydimethylcyclosiloxane [M+K]+"
)
=#
const reference_peaks = Dict(
# DHB Matrix peaks (should be present)
137.0244 => "DHB_fragment",
155.0349 => "DHB_M+H",
177.0168 => "DHB_M+Na",
# Common lipids in your mass range
496.3398 => "PC_16:0_16:0",
520.3398 => "PC_16:0_18:1",
760.5851 => "PC_16:0_18:1_Na",
# Common contaminants
391.2843 => "PDMS",
413.2662 => "PDMS_Na",
# Add some high mass peaks
842.5092 => "Protein_standard",
1045.532 => "Protein_standard",
)
# ===================================================================
# HELPER FUNCTIONS FOR PRINTING
# ===================================================================
function print_header(title::String)
println("\n" * "="^80)
println("$(title)")
println("="^80)
end
function print_subheader(title::String)
println("\n" * "-"^80)
println("$(title)")
println("-"^80)
end
function print_param(key, value)
if value === nothing || (isa(value, Number) && isnan(value))
println(" " * "" * rpad(key, 30) * ": unable to get, user needs to input manually")
elseif isa(value, Symbol) && startswith(string(key), "method") # Heuristic for method selection
println(" " * "" * rpad(key, 30) * ": automatically detected this as the most optimal method: $(value)")
else
println(" " * "" * rpad(key, 30) * ": $(value)")
end
end
function print_recommendations(recommendations::Dict)
for (step, params) in recommendations
print_subheader("Recommendations for $(String(step))")
for (key, value) in params
print_param(key, value)
end
end
end
function check_data_range(msi_data::MSIData)
println("\n--- Data Range Analysis ---")
min_mz, max_mz = get_global_mz_range(msi_data)
if isfinite(min_mz) && isfinite(max_mz) && min_mz < max_mz
println("Global m/z range: [$(min_mz), $(max_mz)]")
else
println("Global m/z range: Not yet determined or invalid (initial: [$(min_mz), $(max_mz)])")
end
# Check a few spectra to see actual m/z values
println("\nChecking first few spectra for actual m/z values:")
for i in 1:min(3, length(msi_data.spectra_metadata))
try
mz, intensity = GetSpectrum(msi_data, i)
if !isempty(mz)
println("Spectrum $i: m/z range [$(minimum(mz)), $(maximum(mz))], length=$(length(mz))")
# Print first and last few m/z values
if length(mz) > 10
println(" First 5 m/z: $(mz[1:5])")
println(" Last 5 m/z: $(mz[end-4:end])")
end
end
catch e
println("Spectrum $i: Error - $e")
end
end
end
# ===================================================================
# MAIN EXAMPLE RUNNER
# ===================================================================
function run_precalculation_example()
# --- Process mzML file ---
print_header("Processing mzML File: $(basename(TEST_MZML_FILE))")
if !isfile(TEST_MZML_FILE)
println("SKIPPING: mzML file not found at $(TEST_MZML_FILE)")
else
try
msi_data_mzml = @time OpenMSIData(TEST_MZML_FILE)
check_data_range(msi_data_mzml)
analysis_results_mzml = run_preprocessing_analysis(msi_data_mzml, reference_peaks=reference_peaks)
println("\n" * "*"^80)
println("MZML PREPROCESSING ANALYSIS RESULTS")
println("*"^80)
for (phase, results) in analysis_results_mzml
if phase == :recommendations
print_subheader("Generated Preprocessing Recommendations")
print_recommendations(results)
else
print_subheader("Phase: $(String(phase))")
if isa(results, Dict)
for (key, value) in results
print_param(key, value)
end
elseif isa(results, NamedTuple)
for field in fieldnames(typeof(results))
print_param(field, getfield(results, field))
end
else
println(" $(String(phase)) results: $(results)")
end
end
end
close(msi_data_mzml) # Close file handles
catch e
println("ERROR processing mzML file: $e")
showerror(stdout, e, catch_backtrace())
end
end
# --- Process imzML file ---
print_header("Processing imzML File: $(basename(TEST_IMZML_FILE))")
if !isfile(TEST_IMZML_FILE)
println("SKIPPING: imzML file not found at $(TEST_IMZML_FILE)")
else
try
msi_data_imzml = @time OpenMSIData(TEST_IMZML_FILE)
check_data_range(msi_data_imzml)
analysis_results_imzml = run_preprocessing_analysis(msi_data_imzml, reference_peaks=reference_peaks, mask_path=MASK_ROUTE)
println("\n" * "*"^80)
println("IMZML PREPROCESSING ANALYSIS RESULTS")
println("*"^80)
for (phase, results) in analysis_results_imzml
if phase == :recommendations
print_subheader("Generated Preprocessing Recommendations")
print_recommendations(results)
else
print_subheader("Phase: $(String(phase))")
if isa(results, Dict)
for (key, value) in results
print_param(key, value)
end
elseif isa(results, NamedTuple)
for field in fieldnames(typeof(results))
print_param(field, getfield(results, field))
end
else
println(" $(String(phase)) results: $(results)")
end
end
end
close(msi_data_imzml) # Close file handles
catch e
println("ERROR processing imzML file: $e")
showerror(stdout, e, catch_backtrace())
end
end
end
# --- Execute ---
@time run_precalculation_example()
# =============================================================================
# Example Usage
# =============================================================================
#=
"""
example_preanalysis_workflow(msi_data::MSIData)
Demonstrates how to run the pre-analysis pipeline.
"""
function example_preanalysis_workflow(msi_data::MSIData)
# Load your MSIData object (this would come from your actual data loading)
# msi_data = load_imzml_dataset("path/to/your/data.imzML") # Assuming msi_data is passed
# Define reference peaks for mass accuracy analysis
reference_peaks = Dict(
89.04767 => "Alanin",
147.07642 => "Lysin",
189.12392 => "Unknown",
524.26496 => "PC(34:1) [M+H]+"
)
# Define region masks if you have spatial annotations
region_masks = Dict{Symbol, BitMatrix}()
# region_masks[:tumor] = load_mask("tumor_mask.png") # Not defined, comment out
# region_masks[:stroma] = load_mask("stroma_mask.png") # Not defined, comment out
println("Starting comprehensive pre-analysis...")
# Run the complete pre-analysis pipeline
analysis_results = run_preprocessing_analysis(
msi_data, # Your MSIData object
reference_peaks=reference_peaks,
region_masks=region_masks,
sample_size=200 # Adjust based on dataset size
)
# Access the recommendations
recommendations = analysis_results[:recommendations]
println("\n" * "="^60)
println("PREPROCESSING RECOMMENDATIONS")
println("="^60)
for (step, params) in recommendations
println("\n$step:")
for (key, value) in params
println(" - $key: $value")
end
end
return analysis_results
end
# You can also run individual analysis steps:
function run_targeted_analysis(msi_data::MSIData)
# Just analyze signal quality and peak characteristics
signal_analysis = analyze_signal_quality(msi_data, sample_size=100)
peak_analysis = analyze_peak_characteristics(msi_data, sample_size=50)
return (signal_analysis, peak_analysis)
end
=#

785
test/run_preprocessing.jl
View File

@ -1,371 +1,534 @@
# test/run_preprocessing.jl # run_preprocessing.jl
# ===================================================================
# Preprocessing Test Suite for JuliaMSI
# ===================================================================
# This script provides a customizable workflow to test and visualize
# the effects of different preprocessing steps on individual spectra
# from .mzML or .imzML files.
#
# Instructions:
# 1. Configure the file paths and parameters in the "CONFIG" section.
# 2. Run the script from the project's root directory:
# julia --threads auto --project=. test/run_preprocessing.jl
# 3. Check the configured `RESULTS_DIR` for output plots and CSV files.
# ===================================================================
using Printf using Printf
using CairoMakie
using DataFrames
using CSV
using Statistics
using Interpolations
import Pkg import Pkg
using CairoMakie
using DataFrames # For creating dataframes
using CSV
# --- Load the MSI_src Module --- # --- Load the MSI_src Module ---
# Activate the project environment at the parent directory of this test script
Pkg.activate(joinpath(@__DIR__, "..")) Pkg.activate(joinpath(@__DIR__, ".."))
using MSI_src using MSI_src
# =================================================================== # ===================================================================
# CONFIG: CUSTOMIZE YOUR PREPROCESSING WORKFLOW HERE # CONFIG: PLEASE FILL IN YOUR FILE PATHS HERE
# =================================================================== # ===================================================================
# --- Input and Output --- const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML" # Can be .mzML or .imzML # const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/set de datos MS/Atropina_tuneo_fraq_20ev.mzML"
const RESULTS_DIR = "test/results/preprocessing" # const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML"
const NUM_SPECTRA_TO_PROCESS = nothing # Set to `nothing` to process all spectra
# --- Internal Standards for Calibration and QC --- # const MASK_ROUTE = "/home/pixel/Documents/Cinvestav_2025/JuliaMSI/public/css/masks/Stomach_DHB_uncompressed.png"
# replace these with m/z values of known compounds present in your dataset. const MASK_ROUTE = ""
const INTERNAL_STANDARDS = Dict(
"P13" => 432.6584,
"P15" => 464.6059,
"P17" => 526.5534,
"P21" => 650.4485,
"P25" => 774.3435,
"P29" => 898.2385,
"P31" => 950.1861,
"P33" => 1022.1336,
"P37" => 1146.0286,
"P45" => 1593.8187,
"Unknown 1" => 772.433,
"Unknown 2" => 772.5253
)
# --- Preprocessing Step Parameters --- const OUTPUT_DIR = "./test/results/preprocessing_results"
# Step 0: Quality Control (QC) # Reference peaks for calibration and alignment
const QC_PARAMS = ( const reference_peaks = Dict(
ppm_tolerance = 5.0, # PPM tolerance for matching internal standards 137.0244 => "DHB_fragment",
) 155.0349 => "DHB_M+H",
177.0168 => "DHB_M+Na",
# Step 1: Calibration 496.3398 => "PC_16:0_16:0",
const CALIBRATION_PARAMS = ( 520.3398 => "PC_16:0_18:1",
enabled = true, 760.5851 => "PC_16:0_18:1_Na",
ppm_tolerance = 5.0, # PPM tolerance for finding calibration peaks 391.2843 => "PDMS",
) 413.2662 => "PDMS_Na",
842.5092 => "Protein_standard",
# Step 2: Smoothing 1045.532 => "Protein_standard",
# Note: Smoothing is less effective and often unnecessary for centroid data. 290.1747 => "Atropine [M+H]+",
const SMOOTHING_PARAMS = ( 304.1903 => "Scopolamine [M+H]+",
enabled = false, 124.0393 => "Tropine [M+H]+",
window = 9,
order = 2,
)
# Step 3: Baseline Correction
# Note: Baseline correction is less effective and often unnecessary for centroid data.
const BASELINE_CORRECTION_PARAMS = (
enabled = false,
iterations = 100,
)
# Step 4: Normalization
const NORMALIZATION_PARAMS = (
enabled = true,
method = :tic, # :tic, :median, or :none
)
# Step 5: Peak Detection
const PEAK_DETECTION_PARAMS = (
enabled = true,
method = :centroid, # Use :centroid for centroided data, :profile for profile data
# --- Parameters for :centroid method ---
intensity_threshold = 0.0, # Filters out peaks below this absolute intensity
# --- Parameters for :profile method ---
half_window = 10,
snr_threshold = 3.0,
min_peak_prominence = 0.05,
merge_peaks_tolerance = 0.002,
)
# Step 6: Peak Alignment (Warping)
# This is performed after collecting peaks from all spectra.
const PEAK_ALIGNMENT_PARAMS = (
enabled = true,
tolerance = 0.002,
tolerance_unit = :mz, # :mz or :ppm
min_matched_peaks = 3, # Lowered for sparse centroid data
)
# Step 7: Peak Binning
const PEAK_BINNING_PARAMS = (
enabled = true,
tolerance = 0.1,
tolerance_unit = :mz, # :mz or :ppm
frequency_threshold = 0.1, # Min fraction of spectra a peak must be in
) )
# =================================================================== # ===================================================================
# HELPER FUNCTIONS # HELPER FUNCTIONS
# =================================================================== # ===================================================================
""" function ensure_output_dir()
plot_spectrum_on_fig(fig_pos, mz, intensity, title) if !isdir(OUTPUT_DIR)
mkdir(OUTPUT_DIR)
Helper function to plot a spectrum on a specific position of a CairoMakie figure. println("Created output directory: $OUTPUT_DIR")
""" end
function plot_spectrum_on_fig(fig_pos, mz, intensity, title)
ax = Axis(fig_pos, title=title, xlabel="m/z", ylabel="Intensity")
lines!(ax, mz, intensity)
end end
function plot_spectrum_step(mz, intensity, step_name; peaks=nothing, spectrum_index=1)
fig = Figure(size=(1200, 600))
ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Intensity", title="Step: $step_name (Spectrum $spectrum_index)")
# =================================================================== lines!(ax, mz, intensity, color=:blue, linewidth=1)
# MAIN PROCESSING SCRIPT
# ===================================================================
function run_preprocessing_suite() if peaks !== nothing && !isempty(peaks)
println("Starting Preprocessing Test Suite...") peak_mz = [p.mz for p in peaks]
mkpath(RESULTS_DIR) peak_intensity = [p.intensity for p in peaks]
scatter!(ax, peak_mz, peak_intensity, color=:red, markersize=5)
# --- Load Data ---
println("Loading data from: $TEST_FILE")
if !isfile(TEST_FILE)
println("ERROR: Test file not found. Please check the path in the CONFIG section.")
return
end end
msi_data = @time OpenMSIData(TEST_FILE)
println("Data loaded successfully. Found $(length(msi_data.spectra_metadata)) spectra.")
# --- Determine which spectra to process --- filename = joinpath(OUTPUT_DIR, "spectrum_$(step_name)_index_$spectrum_index.png")
total_spectra = length(msi_data.spectra_metadata) save(filename, fig)
indices_to_process = if NUM_SPECTRA_TO_PROCESS === nothing println(" - Saved plot: $(basename(filename))")
1:total_spectra end
function print_step_header(step_name::String)
println("\n" * ">"^50)
println("APPLYING STEP: $step_name")
println(">"^50)
end
# ===================================================================
# PREPROCESSING PIPELINE FUNCTIONS (IN-PLACE)
# ===================================================================
function apply_baseline_correction(spectra::Vector{MutableSpectrum}, params::Dict, msi_data::MSIData)
print_step_header("Baseline Correction")
method = get(params, :method, :snip)
iterations = get(params, :iterations, 100)
window = get(params, :window, 20)
println(" Method: $method, Iterations: $iterations, Window: $window")
# Safely plot the first spectrum
if !isempty(spectra)
s = spectra[1]
if validate_spectrum(s.mz, s.intensity)
baseline = MSI_src.apply_baseline_correction(s.intensity; method=method, iterations=iterations, window=window)
corrected_intensity = max.(0.0, s.intensity .- baseline)
plot_spectrum_step(s.mz, corrected_intensity, "baseline_correction")
end
end
Threads.@threads for s in spectra
if validate_spectrum(s.mz, s.intensity)
baseline = MSI_src.apply_baseline_correction(s.intensity; method=method, iterations=iterations, window=window)
s.intensity = max.(0.0, s.intensity .- baseline)
end
end
end
function apply_smoothing(spectra::Vector{MutableSpectrum}, params::Dict)
print_step_header("Smoothing")
method = get(params, :method, :savitzky_golay)
window = get(params, :window, 9)
order = get(params, :order, 2)
println(" Method: $method, Window: $window, Order: $order")
# Safely plot the first spectrum
if !isempty(spectra)
s = spectra[1]
if validate_spectrum(s.mz, s.intensity)
smoothed_intensity = max.(0.0, smooth_spectrum(s.intensity; method=method, window=window, order=order))
plot_spectrum_step(s.mz, smoothed_intensity, "smoothing", spectrum_index=s.id)
end
end
Threads.@threads for s in spectra
if validate_spectrum(s.mz, s.intensity)
smoothed_intensity = max.(0.0, smooth_spectrum(s.intensity; method=method, window=window, order=order))
s.intensity = smoothed_intensity
end
end
end
function apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
print_step_header("Peak Picking")
method = get(params, :method, :profile)
snr_threshold = get(params, :snr_threshold, 3.0)
half_window = get(params, :half_window, 10)
min_peak_prominence = get(params, :min_peak_prominence, 0.1)
merge_peaks_tolerance = get(params, :merge_peaks_tolerance, 0.002)
println(" Method: $method, SNR: $snr_threshold, Half Window: $half_window")
found_invalid_spectrum = Threads.Atomic{Bool}(false)
Threads.@threads for s in spectra
is_valid = validate_spectrum(s.mz, s.intensity)
if !is_valid && !found_invalid_spectrum[]
if Threads.atomic_xchg!(found_invalid_spectrum, true) == false
# Debug printing for the first invalid spectrum found
end
end
if is_valid
if method == :profile
s.peaks = detect_peaks_profile(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window, min_peak_prominence=min_peak_prominence, merge_peaks_tolerance=merge_peaks_tolerance)
elseif method == :wavelet
s.peaks = detect_peaks_wavelet(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
elseif method == :centroid
s.peaks = detect_peaks_centroid(s.mz, s.intensity; snr_threshold=snr_threshold)
else else
unique(round.(Int, range(1, total_spectra, length=min(NUM_SPECTRA_TO_PROCESS, total_spectra)))) s.peaks = detect_peaks_profile(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
end
else
s.peaks = []
end
end end
println("Will process $(length(indices_to_process)) spectra.")
# --- Data storage for results --- # Safely plot the first spectrum
qc_results = DataFrame(spectrum_idx=Int[], metric=String[], value=Float64[], compound=String[]) if !isempty(spectra)
all_processed_peaks_mz = Vector{Vector{Float64}}() s1 = findfirst(s -> s.id == 1, spectra)
all_processed_peaks_int = Vector{Vector{Float64}}() if s1 !== nothing
spectrum_indices_with_peaks = Int[] plot_spectrum_step(spectra[s1].mz, spectra[s1].intensity, "peak_picking", peaks=spectra[s1].peaks)
println(" - Detected $(length(spectra[s1].peaks)) peaks in spectrum 1")
end
end
end
# --- Main Loop: Process each spectrum individually --- function apply_calibration(spectra::Vector{MutableSpectrum}, params::Dict, reference_peaks::Dict)
println("\n" * "="^20 * " Processing Individual Spectra " * "="^20) print_step_header("Calibration")
for (i, idx) in enumerate(indices_to_process)
print("\rProcessing spectrum #$idx ($(i)/$(length(indices_to_process)))...")
process_spectrum(msi_data, idx) do mz, intensity method = get(params, :method, :none)
if qc_is_empty(mz, intensity) || !qc_is_regular(mz) ppm_tolerance = get(params, :ppm_tolerance, 20.0)
# @warn "Skipping empty or irregular spectrum #$idx" println(" Method: $method, PPM Tolerance: $ppm_tolerance")
if method == :none || isempty(reference_peaks)
println(" - Skipping calibration (no method or reference peaks)")
return return
end end
original_mz, original_intensity = copy(mz), copy(intensity) reference_masses = collect(keys(reference_peaks))
processed_mz, processed_intensity = copy(mz), copy(intensity) calibration_info = Vector{String}(undef, length(spectra))
# --- Step 0: QC (PPM and Resolution) --- Threads.@threads for i in 1:length(spectra)
# This happens before any modification to the m/z axis s = spectra[i]
let ref_masses = collect(values(INTERNAL_STANDARDS)) info_message = ""
matched_peaks = find_calibration_peaks(processed_mz, processed_intensity, ref_masses, ppm_tolerance=QC_PARAMS.ppm_tolerance) if validate_spectrum(s.mz, s.intensity)
matched_peaks = find_calibration_peaks(s.mz, s.intensity, reference_masses; ppm_tolerance=ppm_tolerance)
for (compound, theoretical_mz) in INTERNAL_STANDARDS
# Find the measured m/z that corresponds to this theoretical_mz
measured_mz = 0.0
for (theo, meas) in matched_peaks
if theo == theoretical_mz
measured_mz = meas
break
end
end
if measured_mz > 0
# PPM Error
ppm_error = calculate_ppm_error(measured_mz, theoretical_mz)
push!(qc_results, (idx, "ppm_error", ppm_error, compound))
# Resolution
resolution = calculate_resolution_fwhm(measured_mz, processed_mz, processed_intensity)
if !isnan(resolution)
push!(qc_results, (idx, "resolution", resolution, compound))
end
end
end
end
# --- Step 1: Calibration ---
if CALIBRATION_PARAMS.enabled
ref_masses = collect(values(INTERNAL_STANDARDS))
matched_peaks = find_calibration_peaks(processed_mz, processed_intensity, ref_masses, ppm_tolerance=CALIBRATION_PARAMS.ppm_tolerance)
if length(matched_peaks) >= 2 if length(matched_peaks) >= 2
measured = sort(collect(values(matched_peaks))) measured = sort(collect(values(matched_peaks)))
theoretical = sort(collect(keys(matched_peaks))) theoretical = sort(collect(keys(matched_peaks)))
itp = linear_interpolation(measured, theoretical, extrapolation_bc=Line()) itp = linear_interpolation(measured, theoretical, extrapolation_bc=Line())
processed_mz = itp(processed_mz) s.mz = itp(s.mz) # Modify mz-axis in-place
info_message = " - Spectrum $(s.id): calibrated using $(length(matched_peaks)) reference peaks"
else
info_message = " - Spectrum $(s.id): insufficient reference peaks ($(length(matched_peaks)) found), skipping"
end
end
calibration_info[i] = info_message
end
# Print summary of results
printed_count = 0
for info in calibration_info
if !isempty(info) && printed_count < 5
println(info)
printed_count += 1
end
end end
end end
# --- Step 2: Smoothing --- function apply_peak_alignment(spectra::Vector{MutableSpectrum}, params::Dict)
if SMOOTHING_PARAMS.enabled print_step_header("Peak Alignment")
processed_intensity = smooth_spectrum(processed_intensity, window=SMOOTHING_PARAMS.window, order=SMOOTHING_PARAMS.order)
end
# --- Step 3: Baseline Correction --- method = get(params, :method, :none)
if BASELINE_CORRECTION_PARAMS.enabled tolerance = get(params, :tolerance, 0.002)
baseline = snip_baseline(processed_intensity, iterations=BASELINE_CORRECTION_PARAMS.iterations) tolerance_unit = get(params, :tolerance_unit, :mz)
processed_intensity .-= baseline println(" Method: $method, Tolerance: $tolerance $tolerance_unit")
processed_intensity = max.(0, processed_intensity) # Ensure non-negativity
end
# --- Step 4: Normalization --- if method == :none
if NORMALIZATION_PARAMS.enabled && NORMALIZATION_PARAMS.method != :none println(" - Skipping peak alignment")
if NORMALIZATION_PARAMS.method == :tic
processed_intensity = tic_normalize(processed_intensity)
elseif NORMALIZATION_PARAMS.method == :median
processed_intensity = median_normalize(processed_intensity)
end
end
# --- Step 5: Peak Detection ---
local pk_mz, pk_int
pk_mz, pk_int = Float64[], Float64[] # Initialize empty
if PEAK_DETECTION_PARAMS.enabled
if PEAK_DETECTION_PARAMS.method == :profile
pk_mz, pk_int = detect_peaks_profile(processed_mz, processed_intensity,
half_window=PEAK_DETECTION_PARAMS.half_window,
snr_threshold=PEAK_DETECTION_PARAMS.snr_threshold,
min_peak_prominence=PEAK_DETECTION_PARAMS.min_peak_prominence,
merge_peaks_tolerance=PEAK_DETECTION_PARAMS.merge_peaks_tolerance)
elseif PEAK_DETECTION_PARAMS.method == :wavelet
pk_mz, pk_int = detect_peaks_wavelet(processed_mz, processed_intensity)
else # :centroid
pk_mz, pk_int = detect_peaks_centroid(processed_mz, processed_intensity)
end
if !isempty(pk_mz)
push!(all_processed_peaks_mz, pk_mz)
push!(all_processed_peaks_int, pk_int)
push!(spectrum_indices_with_peaks, idx)
end
end
# --- Visualization of a sample spectrum ---
if i == 1 # Only plot the first processed spectrum
println("\nGenerating example plots for spectrum #$idx...")
fig = Figure(size=(1200, 800))
plot_spectrum_on_fig(fig[1,1], original_mz, original_intensity, "1. Original Spectrum")
plot_spectrum_on_fig(fig[2,1], processed_mz, processed_intensity, "2. After All Steps (Before Peak Picking)")
# Plot detected peaks
ax = Axis(fig[3,1], title="3. Detected Peaks")
if !isempty(pk_mz)
stem!(ax, pk_mz, pk_int)
end
save(joinpath(RESULTS_DIR, "example_spectrum_processing.png"), fig)
println("Saved example processing plots.")
end
end
end
println("\nIndividual spectrum processing complete.")
# --- Save QC Results ---
if !isempty(qc_results)
println("\n" * "="^20 * " Generating QC Report " * "="^20)
CSV.write(joinpath(RESULTS_DIR, "qc_results.csv"), qc_results)
println("QC results saved to qc_results.csv")
# Generate summary plots for QC
fig = Figure(size=(1200, 600))
# PPM Error Histogram
ppm_errors = filter(row -> row.metric == "ppm_error", qc_results).value
if !isempty(ppm_errors)
ax1 = Axis(fig[1,1], title="PPM Error Distribution", xlabel="PPM Error")
hist!(ax1, ppm_errors, bins=30)
end
# Resolution Histogram
resolutions = filter(row -> row.metric == "resolution", qc_results).value
if !isempty(resolutions)
ax2 = Axis(fig[1,2], title="Resolution Distribution", xlabel="Resolution (FWHM)")
hist!(ax2, resolutions, bins=30)
end
save(joinpath(RESULTS_DIR, "qc_summary_plots.png"), fig)
println("QC summary plots saved.")
end
if isempty(all_processed_peaks_mz)
@warn "No peaks were detected in any of the processed spectra. Skipping alignment and binning."
return return
end end
# --- Step 6: Peak Alignment --- ref_find_idx = findfirst(s -> !isempty(s.peaks), spectra)
println("\n" * "="^20 * " Aligning Peaks " * "="^20) if ref_find_idx === nothing
aligned_peaks_mz = copy(all_processed_peaks_mz) println(" - Insufficient spectra with peaks for alignment. Skipping.")
if PEAK_ALIGNMENT_PARAMS.enabled return
# Create a reference peak list (e.g., from the spectrum with the most peaks)
ref_idx = argmax(length.(all_processed_peaks_mz))
reference_peaks = all_processed_peaks_mz[ref_idx]
println("Using spectrum $(spectrum_indices_with_peaks[ref_idx]) as alignment reference.")
for i in 1:length(aligned_peaks_mz)
if i == ref_idx continue end
alignment_func = align_peaks_lowess(reference_peaks, aligned_peaks_mz[i],
tolerance=PEAK_ALIGNMENT_PARAMS.tolerance,
tolerance_unit=PEAK_ALIGNMENT_PARAMS.tolerance_unit,
min_matched_peaks=PEAK_ALIGNMENT_PARAMS.min_matched_peaks)
aligned_peaks_mz[i] = alignment_func(aligned_peaks_mz[i])
end
println("Peak alignment complete.")
end end
# --- Step 7: Peak Binning & Feature Matrix Generation --- ref_spectrum = spectra[ref_find_idx]
println("\n" * "="^20 * " Binning Peaks " * "="^20) ref_peaks_mz = [p.mz for p in ref_spectrum.peaks]
if PEAK_BINNING_PARAMS.enabled println(" - Using spectrum $(ref_spectrum.id) as reference with $(length(ref_peaks_mz)) peaks")
feature_matrix, mz_bins = bin_peaks(aligned_peaks_mz, all_processed_peaks_int,
PEAK_BINNING_PARAMS.tolerance,
tolerance_unit=PEAK_BINNING_PARAMS.tolerance_unit,
frequency_threshold=PEAK_BINNING_PARAMS.frequency_threshold)
if !isempty(feature_matrix) Threads.@threads for s in spectra
df = DataFrame(feature_matrix, :auto) if s.id == ref_spectrum.id || isempty(s.peaks)
rename!(df, ["bin_$(i)" for i in 1:size(df, 2)]) continue
insertcols!(df, 1, :spectrum_idx => spectrum_indices_with_peaks) end
CSV.write(joinpath(RESULTS_DIR, "feature_matrix.csv"), df) current_peaks_mz = [p.mz for p in s.peaks]
println("Feature matrix saved to feature_matrix.csv") alignment_func = align_peaks_lowess(ref_peaks_mz, current_peaks_mz; method=method, tolerance=tolerance, tolerance_unit=tolerance_unit)
# Save bin m/z ranges s.mz = alignment_func.(s.mz) # Update m/z axis
bin_df = DataFrame(bin_index=1:length(mz_bins), mz_start=[b[1] for b in mz_bins], mz_end=[b[2] for b in mz_bins])
CSV.write(joinpath(RESULTS_DIR, "bin_definitions.csv"), bin_df) # Update peak m/z values
println("Bin definitions saved to bin_definitions.csv") for i in 1:length(s.peaks)
old_peak = s.peaks[i]
aligned_peak_mz = alignment_func(old_peak.mz)
s.peaks[i] = (mz=aligned_peak_mz, intensity=old_peak.intensity, fwhm=old_peak.fwhm, shape_r2=old_peak.shape_r2, snr=old_peak.snr, prominence=old_peak.prominence)
end
end
end
function apply_normalization(spectra::Vector{MutableSpectrum}, params::Dict)
print_step_header("Normalization")
method = get(params, :method, :tic)
println(" Method: $method")
# Safely plot the first spectrum
if !isempty(spectra)
s1 = findfirst(s -> s.id == 1, spectra)
if s1 !== nothing
s = spectra[s1]
if validate_spectrum(s.mz, s.intensity)
normalized_intensity = MSI_src.apply_normalization(s.intensity; method=method)
plot_spectrum_step(s.mz, normalized_intensity, "normalization")
end
end
end
Threads.@threads for s in spectra
if validate_spectrum(s.mz, s.intensity)
s.intensity = MSI_src.apply_normalization(s.intensity; method=method)
end
end
end
function apply_peak_binning(spectra::Vector{MutableSpectrum}, params::Dict)
print_step_header("Peak Binning")
method = get(params, :method, :adaptive)
tolerance = get(params, :tolerance, 20.0)
tolerance_unit = get(params, :tolerance_unit, :ppm)
min_peak_per_bin = get(params, :min_peak_per_bin, 3)
max_bin_width_ppm = get(params, :max_bin_width_ppm, 150.0)
intensity_weighted_centers = get(params, :intensity_weighted_centers, true)
println(" Method: $method, Tolerance: $tolerance $tolerance_unit")
if isempty(spectra) || all(s -> isempty(s.peaks), spectra)
@warn "No peaks found for binning. Returning empty feature matrix."
return nothing, nothing
end
println(" - Binning peaks from $(length(spectra)) spectra")
binning_params = PeakBinningParams(
method=method,
tolerance=tolerance,
tolerance_unit=tolerance_unit,
min_peak_per_bin=min_peak_per_bin,
max_bin_width_ppm=max_bin_width_ppm,
intensity_weighted_centers=intensity_weighted_centers
)
feature_matrix, bin_definitions = bin_peaks(spectra, binning_params)
if feature_matrix !== nothing && bin_definitions !== nothing
println(" - Generated feature matrix: $(size(feature_matrix.matrix))")
println(" - Number of bins: $(length(bin_definitions))")
end
return feature_matrix, bin_definitions
end
function save_feature_matrix(feature_matrix, bin_definitions)
print_step_header("Saving Results")
# Save feature matrix as CSV
csv_path = joinpath(OUTPUT_DIR, "feature_matrix.csv")
# Create column headers
bin_headers = ["bin_$(i)_$(round(def[1], digits=4))-$(round(def[2], digits=4))"
for (i, def) in enumerate(bin_definitions)]
# Open file for writing
open(csv_path, "w") do io
# Write header row
write(io, "spectrum_index," * join(bin_headers, ",") * "\n")
# Write data rows
for r_idx in 1:size(feature_matrix.matrix, 1)
write(io, "$(feature_matrix.sample_ids[r_idx]),")
for c_idx in 1:size(feature_matrix.matrix, 2)
write(io, "$(feature_matrix.matrix[r_idx, c_idx])")
if c_idx < size(feature_matrix.matrix, 2)
write(io, ",")
end
end
write(io, "\n")
end
end
println(" - Saved feature matrix: $csv_path")
# Save bin definitions (still using DataFrame as it's small)
bins_path = joinpath(OUTPUT_DIR, "bin_definitions.csv")
bins_df = DataFrame(
bin_index = 1:length(bin_definitions),
mz_start = [def[1] for def in bin_definitions],
mz_end = [def[2] for def in bin_definitions],
mz_center = [(def[1] + def[2])/2 for def in bin_definitions]
)
CSV.write(bins_path, bins_df)
println(" - Saved bin definitions: $bins_path")
return csv_path, bins_path
end
# ===================================================================
# USER OVERRIDES: Manually specify parameters here
# ===================================================================
# This dictionary allows you to override any auto-detected parameters.
# The structure should match the output of `main_precalculation`.
# Example: Force a less aggressive peak prominence threshold.
const USER_OVERRIDES = Dict(
:PeakPicking => Dict(
#:snr_threshold => 1.5,
:half_window => 5,
:snr_threshold => 15.0,
#:merge_peaks_tolerance => 2.5,
#:half_window => 2
),
# :Smoothing => Dict(
# :window => 11
# )
#:PeakAlignment => Dict(
# :tolerance => 0.01
#)
:PeakBinningParams => Dict(
:tolerance => 30.0
)
)
# ===================================================================
# MAIN PREPROCESSING PIPELINE
# ===================================================================
function run_preprocessing_pipeline()
println("="^80)
println("MSI PREPROCESSING PIPELINE")
println("="^80)
# Ensure output directory exists
ensure_output_dir()
# Load data
println("\nLoading data: $(basename(TEST_FILE))")
msi_data = OpenMSIData(TEST_FILE)
# Precompute analytics
println("\nPrecomputing analytics...")
precompute_analytics(msi_data)
# Run precalculation to get automatic parameters
println("\nRunning precalculation for automatic parameter determination...")
auto_params = main_precalculation(msi_data, reference_peaks=reference_peaks, mask_path=MASK_ROUTE)
# --- Apply User Overrides ---
if !isempty(USER_OVERRIDES)
println("\nApplying user overrides...")
for (step, params) in USER_OVERRIDES
if haskey(auto_params, step)
for (param, value) in params
@info "OVERRIDE: For step `:$step`, setting `:$param` to `$value` (was `$(get(auto_params[step], param, "not set"))`)"
end
merge!(auto_params[step], params)
else else
@warn "Feature matrix was empty after binning." @warn "User override for non-existent step `:$step` ignored."
end
end end
end end
println("\nPreprocessing Test Suite finished successfully!") # --- Final Parameter Summary ---
println("\n" * "-"^60)
println("FINAL PARAMETERS FOR PIPELINE RUN")
println("-"^60)
for (step, params) in sort(collect(pairs(auto_params)), by=p -> p.first)
println(" - Step: $step")
if isempty(params)
println(" (No parameters)")
continue
end
for (param, value) in sort(collect(pairs(params)), by=p -> p.first)
println(" - $(rpad(param, 25)): $value")
end
end end
# Define pipeline steps
pipeline_steps = [
"baseline_correction",
"smoothing",
"peak_picking",
"calibration",
"peak_alignment",
"normalization",
"peak_binning"
]
# --- Execute --- println("\nPipeline steps: $(join(pipeline_steps, " -> "))")
run_preprocessing_suite()
# Initialize `current_spectra` as a Vector of mutable structs for in-place modification
println("\nInitializing spectra data structure...")
num_spectra = length(msi_data.spectra_metadata)
current_spectra = Vector{MutableSpectrum}(undef, num_spectra)
_iterate_spectra_fast(msi_data) do idx, mz, intensity
current_spectra[idx] = MutableSpectrum(idx, mz, intensity, [])
# Plot raw spectrum without masks
if idx == 1
plot_spectrum_step(mz, intensity, "raw_unmasked_spectrum")
end
end
# These variables will be populated by the pipeline steps
feature_matrix = nothing
bin_definitions = nothing
# Apply pipeline steps, modifying `current_spectra` in-place
for step in pipeline_steps
println("\n" * "-"^60)
println("PROCESSING STEP: $step")
println("-"^60)
if step == "baseline_correction"
@time apply_baseline_correction(current_spectra, auto_params[:BaselineCorrection], msi_data)
elseif step == "smoothing"
apply_smoothing(current_spectra, auto_params[:Smoothing])
elseif step == "peak_picking"
@time apply_peak_picking(current_spectra, auto_params[:PeakPicking])
elseif step == "calibration"
@time apply_calibration(current_spectra, auto_params[:Calibration], reference_peaks)
elseif step == "peak_alignment"
@time apply_peak_alignment(current_spectra, auto_params[:PeakAlignment])
elseif step == "normalization"
@time apply_normalization(current_spectra, auto_params[:Normalization])
elseif step == "peak_binning"
# This step is different as it generates the final matrix, not modifying spectra in-place
feature_matrix, bin_definitions = @time apply_peak_binning(current_spectra, auto_params[:PeakBinningParams])
if feature_matrix !== nothing
@time save_feature_matrix(feature_matrix, bin_definitions)
end
else
@warn "Unknown step: $step, skipping"
end
# Print progress
println("✓ Completed step: $step")
end
# Clean up
close(msi_data)
GC.gc()
println("\n" * "="^80)
println("PREPROCESSING PIPELINE COMPLETED SUCCESSFULLY!")
println("Results saved to: $OUTPUT_DIR")
println("="^80)
end
# ===================================================================
# EXECUTE PIPELINE
# ===================================================================
if abspath(PROGRAM_FILE) == @__FILE__
@time run_preprocessing_pipeline()
end

6
test/run_tests.jl
View File

@ -75,9 +75,9 @@ const COORDS_TO_PLOT = (50, 50) # Example coordinates (X, Y)
# --- Output Directory --- # --- Output Directory ---
const RESULTS_DIR = "test/results" const RESULTS_DIR = "test/results"
test1 = false test1 = true
test2 = true test2 = false
test3 = false test3 = true
# =================================================================== # ===================================================================
# DATA VALIDATION UTILITY # DATA VALIDATION UTILITY