Fixed preprocessing module, implemented precalculations, this code gives a prediction of the values for the parameters in every step for preprocessing, added test suites for both the parameter generation and the preprocessing pipeline, included a descriptive processing that returns an image with what every parameter does in every step, added instrument metadata structure with the parser for each type of file, to retrieve several important parameters to automate precalculations and preprocessing
This commit is contained in:
Pixelguy14
parent
8fa92dc0e5
commit
bdd9315796
@ -2,7 +2,7 @@
|
||||
|
||||
julia_version = "1.11.7"
|
||||
manifest_format = "2.0"
|
||||
project_hash = "7843f141174fe17a3abc4f25c3819d3195459b5a"
|
||||
project_hash = "4955dfdbb7a233ddefa69e3cce2833f416ffe1bc"
|
||||
|
||||
[[deps.ANSIColoredPrinters]]
|
||||
git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c"
|
||||
|
||||
@ -35,11 +35,13 @@ Loess = "4345ca2d-374a-55d4-8d30-97f9976e7612"
|
||||
Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
|
||||
NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4"
|
||||
NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85"
|
||||
Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a"
|
||||
PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5"
|
||||
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
|
||||
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
|
||||
SavitzkyGolay = "c4bf5708-b6a6-4fbe-bcd0-6850ed671584"
|
||||
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
|
||||
Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
|
||||
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
|
||||
StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
|
||||
StipplePlotly = "ec984513-233d-481d-95b0-a3b58b97af2b"
|
||||
|
||||
167
src/MSIData.jl
167
src/MSIData.jl
@ -6,7 +6,7 @@ including caching and iteration logic, for handling large mzML and imzML dataset
|
||||
efficiently.
|
||||
"""
|
||||
|
||||
using Base64, Libz, Serialization, Printf, DataFrames, Base.Threads # For reading binary data
|
||||
using Base64, Libz, Serialization, Printf, DataFrames, Base.Threads, StatsBase # For reading binary data
|
||||
|
||||
const FILE_HANDLE_LOCK = ReentrantLock()
|
||||
|
||||
@ -28,9 +28,16 @@ function ThreadSafeFileHandle(path::String, mode::String="r")
|
||||
return ThreadSafeFileHandle(path, handle, ReentrantLock())
|
||||
end
|
||||
|
||||
function Base.seek(tsfh::ThreadSafeFileHandle, pos::Integer)
|
||||
"""
|
||||
read_at!(tsfh::ThreadSafeFileHandle, a::AbstractArray, pos::Integer)
|
||||
|
||||
Atomically seeks to a position and reads data into an array. This is the thread-safe
|
||||
way to read from a specific offset in the file.
|
||||
"""
|
||||
function read_at!(tsfh::ThreadSafeFileHandle, a::AbstractArray, pos::Integer)
|
||||
lock(tsfh.lock) do
|
||||
seek(tsfh.handle, pos)
|
||||
read!(tsfh.handle, a)
|
||||
end
|
||||
end
|
||||
|
||||
@ -40,12 +47,6 @@ function Base.read(tsfh::ThreadSafeFileHandle, n::Integer)
|
||||
end
|
||||
end
|
||||
|
||||
function Base.read!(tsfh::ThreadSafeFileHandle, a::AbstractArray)
|
||||
lock(tsfh.lock) do
|
||||
return read!(tsfh.handle, a)
|
||||
end
|
||||
end
|
||||
|
||||
function Base.close(tsfh::ThreadSafeFileHandle)
|
||||
close(tsfh.handle)
|
||||
end
|
||||
@ -66,6 +67,20 @@ function Base.filesize(tsfh::ThreadSafeFileHandle)
|
||||
end
|
||||
end
|
||||
|
||||
# Overload Base.seek for ThreadSafeFileHandle
|
||||
function Base.seek(tsfh::ThreadSafeFileHandle, pos::Integer)
|
||||
lock(tsfh.lock) do
|
||||
seek(tsfh.handle, pos)
|
||||
end
|
||||
end
|
||||
|
||||
# Overload Base.read! for ThreadSafeFileHandle
|
||||
function Base.read!(tsfh::ThreadSafeFileHandle, a::AbstractArray)
|
||||
lock(tsfh.lock) do
|
||||
read!(tsfh.handle, a)
|
||||
end
|
||||
end
|
||||
|
||||
# Abstract type for different data sources (e.g., mzML, imzML)
|
||||
# This allows dispatching to the correct binary reading logic.
|
||||
abstract type MSDataSource end
|
||||
@ -131,6 +146,50 @@ An enum representing the type of a spectrum's data.
|
||||
"""
|
||||
@enum SpectrumMode CENTROID=1 PROFILE=2 UNKNOWN=3
|
||||
|
||||
"""
|
||||
InstrumentMetadata
|
||||
|
||||
Contains metadata about the instrument configuration and acquisition parameters.
|
||||
This information is parsed from the file header and is crucial for guiding
|
||||
preprocessing steps.
|
||||
|
||||
# Fields
|
||||
- `resolution`: Instrument resolution at a specific m/z, if available.
|
||||
- `acquisition_mode`: The overall data acquisition mode (`:profile`, `:centroid`, or `:mixed`).
|
||||
- `mz_axis_type`: The nature of the m/z axis (`:regular`, `:irregular`, `:mixed`).
|
||||
- `calibration_status`: The calibration state of the data (`:internal`, `:external`, `:uncalibrated`).
|
||||
- `instrument_model`: The vendor and model of the instrument.
|
||||
- `mass_accuracy_ppm`: Manufacturer-specified mass accuracy in parts-per-million.
|
||||
- `laser_settings`: A dictionary of MSI-specific laser parameters.
|
||||
- `polarity`: The polarity of the acquisition (`:positive`, `:negative`, `:unknown`).
|
||||
"""
|
||||
struct InstrumentMetadata
|
||||
resolution::Union{Float64, Nothing}
|
||||
acquisition_mode::Symbol
|
||||
mz_axis_type::Symbol
|
||||
calibration_status::Symbol
|
||||
instrument_model::String
|
||||
mass_accuracy_ppm::Union{Float64, Nothing}
|
||||
laser_settings::Union{Dict, Nothing}
|
||||
polarity::Symbol
|
||||
vendor_preprocessing_steps::Union{Vector{String}, Nothing} # New field for provenance
|
||||
end
|
||||
|
||||
# Default constructor for initializing with unknown values
|
||||
function InstrumentMetadata()
|
||||
return InstrumentMetadata(
|
||||
nothing, # resolution
|
||||
:unknown, # acquisition_mode
|
||||
:unknown, # mz_axis_type
|
||||
:uncalibrated, # calibration_status
|
||||
"Not Available",# instrument_model
|
||||
nothing, # mass_accuracy_ppm
|
||||
nothing, # laser_settings
|
||||
:unknown, # polarity
|
||||
nothing # vendor_preprocessing
|
||||
)
|
||||
end
|
||||
|
||||
"""
|
||||
SpectrumMetadata
|
||||
|
||||
@ -150,6 +209,7 @@ struct SpectrumMetadata
|
||||
|
||||
# For mzML
|
||||
id::String
|
||||
type::Symbol # NEW: e.g., :sample, :blank, :qc
|
||||
|
||||
mode::SpectrumMode
|
||||
# Common binary data info
|
||||
@ -180,6 +240,7 @@ efficient repeated access to spectra.
|
||||
mutable struct MSIData
|
||||
source::MSDataSource
|
||||
spectra_metadata::Vector{SpectrumMetadata}
|
||||
instrument_metadata::Union{InstrumentMetadata, Nothing}
|
||||
image_dims::Tuple{Int, Int}
|
||||
coordinate_map::Union{Matrix{Int}, Nothing}
|
||||
|
||||
@ -198,13 +259,14 @@ mutable struct MSIData
|
||||
spectrum_stats_df::Union{DataFrame, Nothing}
|
||||
bloom_filters::Union{Vector{<:BloomFilter}, Nothing}
|
||||
analytics_ready::AtomicFlag
|
||||
preprocessing_hints::Union{Dict{Symbol, Any}, Nothing} # For auto-determined parameters
|
||||
|
||||
function MSIData(source, metadata, dims, coordinate_map, cache_size)
|
||||
obj = new(source, metadata, dims, coordinate_map,
|
||||
function MSIData(source, metadata, instrument_meta, dims, coordinate_map, cache_size)
|
||||
obj = new(source, metadata, instrument_meta, dims, coordinate_map,
|
||||
Dict(), [], cache_size, ReentrantLock(),
|
||||
SimpleBufferPool(),
|
||||
Threads.Atomic{Float64}(Inf), Threads.Atomic{Float64}(-Inf),
|
||||
nothing, nothing, AtomicFlag())
|
||||
nothing, nothing, AtomicFlag(), nothing)
|
||||
|
||||
# Ensure file handles are closed when the object is garbage collected
|
||||
finalizer(obj) do o
|
||||
@ -407,16 +469,17 @@ function read_binary_vector(data::MSIData, io::IO, asset::SpectrumAsset)
|
||||
|
||||
# Calculate number of elements
|
||||
n_elements = length(decoded_bytes) ÷ sizeof(asset.format)
|
||||
out_array = Vector{asset.format}(undef, n_elements)
|
||||
|
||||
# Use unsafe_wrap to avoid copy - this is the critical optimization
|
||||
temp_array = unsafe_wrap(Vector{asset.format}, pointer(decoded_bytes), n_elements)
|
||||
|
||||
# Convert byte order in-place
|
||||
@inbounds for i in 1:n_elements
|
||||
out_array[i] = ltoh(temp_array[i])
|
||||
if n_elements * sizeof(asset.format) != length(decoded_bytes)
|
||||
throw(FileFormatError("Size of decoded byte array is not a multiple of the element size."))
|
||||
end
|
||||
|
||||
# Reinterpret the byte array as an array of the target type. This does not copy.
|
||||
reinterpreted_array = reinterpret(asset.format, decoded_bytes)
|
||||
|
||||
# Allocate the final output array and convert byte order while copying.
|
||||
out_array = [ltoh(x) for x in reinterpreted_array]
|
||||
|
||||
return out_array
|
||||
end
|
||||
|
||||
@ -447,23 +510,13 @@ function read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata)
|
||||
mz = Array{source.mz_format}(undef, meta.mz_asset.encoded_length)
|
||||
intensity = Array{source.intensity_format}(undef, meta.int_asset.encoded_length)
|
||||
|
||||
# FIX: Read arrays in the order they appear in the file for efficiency,
|
||||
# but ensure we seek to the correct offset for both.
|
||||
if meta.mz_asset.offset < meta.int_asset.offset
|
||||
seek(source.ibd_handle, meta.mz_asset.offset)
|
||||
read!(source.ibd_handle, mz)
|
||||
seek(source.ibd_handle, meta.int_asset.offset)
|
||||
read!(source.ibd_handle, intensity)
|
||||
else
|
||||
seek(source.ibd_handle, meta.int_asset.offset)
|
||||
read!(source.ibd_handle, intensity)
|
||||
seek(source.ibd_handle, meta.mz_asset.offset)
|
||||
read!(source.ibd_handle, mz)
|
||||
end
|
||||
# Use the new atomic read_at! method for thread-safety
|
||||
read_at!(source.ibd_handle, mz, meta.mz_asset.offset)
|
||||
read_at!(source.ibd_handle, intensity, meta.int_asset.offset)
|
||||
|
||||
# imzML data is little-endian. Convert to host byte order.
|
||||
mz .= ltoh.(mz)
|
||||
intensity .= ltoh.(intensity) # FIX: Added missing conversion for intensity
|
||||
intensity .= ltoh.(intensity)
|
||||
|
||||
validate_spectrum_data(mz, intensity, meta.id)
|
||||
|
||||
@ -733,6 +786,25 @@ function precompute_analytics(msi_data::MSIData)
|
||||
g_min_mz = minimum(thread_local_min_mz)
|
||||
g_max_mz = maximum(thread_local_max_mz)
|
||||
|
||||
# Only update if we found valid ranges
|
||||
if isfinite(g_min_mz) && isfinite(g_max_mz) && g_min_mz < g_max_mz
|
||||
Threads.atomic_xchg!(msi_data.global_min_mz, g_min_mz)
|
||||
Threads.atomic_xchg!(msi_data.global_max_mz, g_max_mz)
|
||||
else
|
||||
# Fallback: calculate from first spectrum
|
||||
try
|
||||
if num_spectra > 0
|
||||
mz, _ = GetSpectrum(msi_data, 1)
|
||||
if !isempty(mz)
|
||||
Threads.atomic_xchg!(msi_data.global_min_mz, minimum(mz))
|
||||
Threads.atomic_xchg!(msi_data.global_max_mz, maximum(mz))
|
||||
end
|
||||
end
|
||||
catch e
|
||||
@warn "Could not determine global m/z range from first spectrum: $e"
|
||||
end
|
||||
end
|
||||
|
||||
# Create results
|
||||
computed_stats_df = DataFrame(
|
||||
SpectrumID = 1:num_spectra,
|
||||
@ -745,10 +817,6 @@ function precompute_analytics(msi_data::MSIData)
|
||||
Mode = modes
|
||||
)
|
||||
|
||||
# Set atomic values
|
||||
Threads.atomic_xchg!(msi_data.global_min_mz, g_min_mz)
|
||||
Threads.atomic_xchg!(msi_data.global_max_mz, g_max_mz)
|
||||
|
||||
# Set other data
|
||||
msi_data.spectrum_stats_df = computed_stats_df
|
||||
msi_data.bloom_filters = bloom_filters # Will be created on-demand
|
||||
@ -1480,30 +1548,3 @@ function get_bloom_filter(data::MSIData, index::Int)::BloomFilter{Int}
|
||||
return data.bloom_filters[index]
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
monitor_memory_usage(phase::String)
|
||||
|
||||
Utility function to print current memory usage (allocated bytes and RSS)
|
||||
at different phases of execution.
|
||||
|
||||
# Arguments
|
||||
- `phase`: A string describing the current phase (e.g., "Initialization", "Processing chunk 1").
|
||||
"""
|
||||
function monitor_memory_usage(phase::String)
|
||||
# Use Julia's internal memory tracking
|
||||
bytes_allocated = Base.gc_bytes()
|
||||
mb_allocated = bytes_allocated / 1024^2
|
||||
|
||||
# Get process memory from system
|
||||
if Sys.islinux()
|
||||
proc_status = read("/proc/self/status", String)
|
||||
vm_rss_match = match(r"VmRSS:\s*(\d+)\s*kB", proc_status)
|
||||
if vm_rss_match !== nothing
|
||||
vm_rss_mb = parse(Int, vm_rss_match[1]) / 1024
|
||||
@printf "[%s] Memory: %.2f MB allocated, %.2f MB RSS\n" phase mb_allocated vm_rss_mb
|
||||
end
|
||||
else
|
||||
@printf "[%s] Memory: %.2f MB allocated\n" phase mb_allocated
|
||||
end
|
||||
end
|
||||
|
||||
@ -7,7 +7,6 @@ export OpenMSIData,
|
||||
GetSpectrum,
|
||||
IterateSpectra,
|
||||
ImportMzmlFile,
|
||||
plot_slices,
|
||||
plot_slice,
|
||||
get_total_spectrum,
|
||||
get_average_spectrum,
|
||||
@ -16,31 +15,34 @@ export OpenMSIData,
|
||||
generate_colorbar_image,
|
||||
process_image_pipeline,
|
||||
load_and_prepare_mask,
|
||||
set_global_mz_range!
|
||||
set_global_mz_range!,
|
||||
get_global_mz_range,
|
||||
MSIData,
|
||||
_iterate_spectra_fast,
|
||||
validate_spectrum
|
||||
|
||||
# Export the public Preprocessing API
|
||||
export FeatureMatrix,
|
||||
run_preprocessing_pipeline,
|
||||
qc_is_empty,
|
||||
qc_is_regular,
|
||||
transform_intensity,
|
||||
smooth_spectrum,
|
||||
snip_baseline,
|
||||
tic_normalize,
|
||||
pqn_normalize,
|
||||
median_normalize,
|
||||
# Export the public preprocessing & precalculations API
|
||||
export run_preprocessing_analysis,
|
||||
main_precalculation,
|
||||
FeatureMatrix,
|
||||
Calibration,
|
||||
Smoothing,
|
||||
BaselineCorrection,
|
||||
Normalization,
|
||||
PeakPicking,
|
||||
PeakBinningParams,
|
||||
get_masked_spectrum_indices,
|
||||
detect_peaks_profile,
|
||||
detect_peaks_wavelet,
|
||||
detect_peaks_centroid,
|
||||
align_peaks_lowess,
|
||||
find_calibration_peaks,
|
||||
smooth_spectrum,
|
||||
apply_baseline_correction,
|
||||
apply_normalization,
|
||||
bin_peaks,
|
||||
plot_stage_spectrum,
|
||||
calculate_ppm_error,
|
||||
calculate_resolution_fwhm,
|
||||
analyze_mass_accuracy,
|
||||
generate_qc_report,
|
||||
get_common_calibration_standards
|
||||
PeakSelection,
|
||||
PeakAlignment,
|
||||
find_calibration_peaks,
|
||||
align_peaks_lowess,
|
||||
MutableSpectrum
|
||||
|
||||
# Include all source files directly into the main module
|
||||
include("BloomFilters.jl")
|
||||
@ -52,6 +54,9 @@ include("imzML.jl")
|
||||
include("MzmlConverter.jl")
|
||||
include("Preprocessing.jl")
|
||||
include("ImageProcessing.jl")
|
||||
include("Precalculations.jl")
|
||||
|
||||
using Setfield # For immutable struct updates
|
||||
|
||||
|
||||
# --- Main Entry Point --- #
|
||||
@ -63,14 +68,28 @@ Opens a .mzML or .imzML file and prepares it for data access.
|
||||
|
||||
This is the main entry point for the new data access API.
|
||||
"""
|
||||
function OpenMSIData(filepath::String; cache_size=300)
|
||||
function OpenMSIData(filepath::String; cache_size=300, spectrum_type_map::Union{Dict{Int, Symbol}, Nothing}=nothing)
|
||||
local msi_data
|
||||
if endswith(lowercase(filepath), ".mzml")
|
||||
return load_mzml_lazy(filepath, cache_size=cache_size)
|
||||
msi_data = load_mzml_lazy(filepath, cache_size=cache_size)
|
||||
elseif endswith(lowercase(filepath), ".imzml")
|
||||
return load_imzml_lazy(filepath, cache_size=cache_size)
|
||||
msi_data = load_imzml_lazy(filepath, cache_size=cache_size)
|
||||
else
|
||||
error("Unsupported file type: $filepath. Please provide a .mzML or .imzML file.")
|
||||
end
|
||||
|
||||
# Apply spectrum type map if provided
|
||||
if spectrum_type_map !== nothing
|
||||
for (idx, type_symbol) in spectrum_type_map
|
||||
if 1 <= idx <= length(msi_data.spectra_metadata)
|
||||
msi_data.spectra_metadata[idx] = @set msi_data.spectra_metadata[idx].type = type_symbol
|
||||
else
|
||||
@warn "Spectrum index $idx out of bounds for spectrum_type_map. Skipping."
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return msi_data
|
||||
end
|
||||
|
||||
end # module MSI_src
|
||||
|
||||
@ -601,7 +601,7 @@ function ConvertMzmlToImzml(source_file::String, target_ibd_file::String, timing
|
||||
close(msi_data)
|
||||
end
|
||||
|
||||
return binary_meta_vec, coords_vec, (width, height), pixel_modes, ibd_uuid
|
||||
return binary_meta_vec, coords_vec, (width, height), pixel_modes, ibd_uuid, msi_data.instrument_metadata
|
||||
end
|
||||
|
||||
"""
|
||||
@ -621,7 +621,7 @@ experiment and data format.
|
||||
# Returns
|
||||
- `true` on success, `false` on failure.
|
||||
"""
|
||||
function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, coords::Vector{Tuple{Int, Int}}, dims::Tuple{Int, Int}, modes::Vector{SpectrumMode}, ibd_uuid::UUID)
|
||||
function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, coords::Vector{Tuple{Int, Int}}, dims::Tuple{Int, Int}, modes::Vector{SpectrumMode}, ibd_uuid::UUID, instrument_meta::InstrumentMetadata)
|
||||
ibd_file = replace(target_file, r"\.imzML$"i => ".ibd")
|
||||
|
||||
if isempty(binary_meta)
|
||||
@ -652,6 +652,15 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
|
||||
<cvParam cvRef="IMS" accession="IMS:1000080" name="mass spectrum"/>
|
||||
<cvParam cvRef="IMS" accession="IMS:1000031" name="processed"/>
|
||||
<cvParam cvRef="IMS" accession="IMS:1000081" name="ibd uuid" value="$(ibd_uuid)"/>
|
||||
$(
|
||||
if instrument_meta.polarity == :positive
|
||||
""" <cvParam cvRef="MS" accession="MS:1000130" name="positive scan"/>"""
|
||||
elseif instrument_meta.polarity == :negative
|
||||
""" <cvParam cvRef="MS" accession="MS:1000129" name="negative scan"/>"""
|
||||
else
|
||||
""
|
||||
end
|
||||
)
|
||||
</fileContent>
|
||||
</fileDescription>
|
||||
""")
|
||||
@ -693,12 +702,35 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
|
||||
""")
|
||||
write(imzml_stream, """ <instrumentConfigurationList count="1">
|
||||
<instrumentConfiguration id="instrument1">
|
||||
$(
|
||||
if instrument_meta.instrument_model != "Not Available"
|
||||
""" <cvParam cvRef="MS" accession="MS:1000031" name="instrument model" value="$(instrument_meta.instrument_model)"/>"""
|
||||
else
|
||||
""
|
||||
end
|
||||
)
|
||||
$(
|
||||
if instrument_meta.resolution !== nothing
|
||||
""" <cvParam cvRef="MS" accession="MS:1001496" name="mass resolving power" value="$(instrument_meta.resolution)"/>"""
|
||||
else
|
||||
""
|
||||
end
|
||||
)
|
||||
<componentList count="3">
|
||||
<source order="1">
|
||||
<cvParam cvRef="MS" accession="MS:1000075" name="MALDI source"/>
|
||||
</source>
|
||||
<analyzer order="2">
|
||||
<cvParam cvRef="MS" accession="MS:1000084" name="time-of-flight"/>
|
||||
$(
|
||||
if instrument_meta.acquisition_mode == :centroid
|
||||
""" <cvParam cvRef="MS" accession="MS:1000127" name="centroid spectrum"/>"""
|
||||
elseif instrument_meta.acquisition_mode == :profile
|
||||
""" <cvParam cvRef="MS" accession="MS:1000128" name="profile spectrum"/>"""
|
||||
else
|
||||
""
|
||||
end
|
||||
)
|
||||
</analyzer>
|
||||
<detector order="3">
|
||||
<cvParam cvRef="MS" accession="MS:1000253" name="electron multiplier"/>
|
||||
@ -708,11 +740,23 @@ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, c
|
||||
</instrumentConfiguration>
|
||||
</instrumentConfigurationList>
|
||||
""")
|
||||
write(imzml_stream, """ <dataProcessingList count="1">
|
||||
write(imzml_stream, """ <dataProcessingList count="$(1 + (instrument_meta.vendor_preprocessing_steps !== nothing ? length(instrument_meta.vendor_preprocessing_steps) : 0))">
|
||||
<dataProcessing id="conversionProcessing">
|
||||
<processingMethod order="1" softwareRef="MSIConverter">
|
||||
<cvParam cvRef="MS" accession="MS:1000544" name="Conversion to imzML"/>
|
||||
</processingMethod>
|
||||
$(
|
||||
if instrument_meta.vendor_preprocessing_steps !== nothing
|
||||
join([
|
||||
""" <processingMethod order="$(i + 1)" softwareRef="MSIConverter">
|
||||
<cvParam cvRef="MS" accession="MS:1000589" name="data processing" value="$(step)"/>
|
||||
</processingMethod>"""
|
||||
for (i, step) in enumerate(instrument_meta.vendor_preprocessing_steps)
|
||||
], "\n")
|
||||
else
|
||||
""
|
||||
end
|
||||
)
|
||||
</dataProcessing>
|
||||
</dataProcessingList>
|
||||
""")
|
||||
@ -812,13 +856,13 @@ function ImportMzmlFile(source_file::String, sync_file::String, target_file::Str
|
||||
|
||||
println("Step 3: Converting spectra and writing .ibd file...")
|
||||
ibd_file = replace(target_file, r"\.imzML$"i => ".ibd")
|
||||
binary_meta, coords, (width, height), pixel_modes, ibd_uuid = ConvertMzmlToImzml(source_file, ibd_file, timing_matrix, scans)
|
||||
binary_meta, coords, (width, height), pixel_modes, ibd_uuid, instrument_meta = ConvertMzmlToImzml(source_file, ibd_file, timing_matrix, scans)
|
||||
|
||||
# Flip image vertically to match R script output
|
||||
flipped_coords = [(x, height - y + 1) for (x, y) in coords]
|
||||
|
||||
println("Step 4: Exporting .imzML metadata file...")
|
||||
success = ExportImzml(target_file, binary_meta, flipped_coords, (width, height), pixel_modes, ibd_uuid)
|
||||
success = ExportImzml(target_file, binary_meta, flipped_coords, (width, height), pixel_modes, ibd_uuid, instrument_meta)
|
||||
|
||||
if success
|
||||
println("Conversion successful: $target_file")
|
||||
|
||||
1537
src/Precalculations.jl
Normal file
1537
src/Precalculations.jl
Normal file
File diff suppressed because it is too large
Load Diff
1563
src/Preprocessing.jl
1563
src/Preprocessing.jl
File diff suppressed because it is too large
Load Diff
@ -1,446 +0,0 @@
|
||||
# src/PreprocessingVersioning.jl
|
||||
|
||||
"""
|
||||
This module provides a versioning helper functions in our data preprocessing, module
|
||||
inspired by the functionality of the R package MALDIquant. It includes
|
||||
functions for quality control, intensity transformation, smoothing, baseline correction,
|
||||
normalization, peak picking, alignment, and feature matrix generation.
|
||||
|
||||
This module is unfinished and may not end up in the release version.
|
||||
"""
|
||||
|
||||
# =============================================================================
|
||||
# Dependencies
|
||||
# =============================================================================
|
||||
|
||||
using UUIDs, JLD2, CodecBase # For preprocessing versioning
|
||||
|
||||
# =============================================================================
|
||||
# Data Structures
|
||||
# =============================================================================
|
||||
|
||||
"""
|
||||
VersionedSpectralData
|
||||
|
||||
A struct to hold a snapshot of spectral data at a specific point in the
|
||||
preprocessing workflow. Each transformation creates a new instance of this
|
||||
struct, forming a history of all operations.
|
||||
|
||||
# Fields
|
||||
- `version_id::String`: A unique identifier for this version of the data.
|
||||
- `parent_id::String`: The identifier of the version from which this one was derived.
|
||||
- `spectra::Vector`: The spectral data itself, typically a vector of `(mz, intensity)` tuples.
|
||||
- `processing_step::String`: A description of the operation that created this version (e.g., "smooth", "baseline").
|
||||
- `timestamp::DateTime`: The time at which this version was created.
|
||||
- `parameters::Dict`: A dictionary of the parameters used in the processing step.
|
||||
"""
|
||||
struct VersionedSpectralData
|
||||
version_id::String
|
||||
parent_id::String
|
||||
spectra::Vector
|
||||
processing_step::String
|
||||
timestamp::DateTime
|
||||
parameters::Dict
|
||||
end
|
||||
|
||||
"""
|
||||
MSISession
|
||||
|
||||
Manages the state of a preprocessing session, including the history of all
|
||||
data versions and the current working version.
|
||||
|
||||
# Fields
|
||||
- `session_id::String`: A unique identifier for the entire session.
|
||||
- `original_data::VersionedSpectralData`: The initial, unprocessed spectral data.
|
||||
- `history::Vector{VersionedSpectralData}`: A chronological list of all data versions created during the session.
|
||||
- `current_version::VersionedSpectralData`: The version of the data currently being worked on or displayed.
|
||||
- `processing_steps::Vector{String}`: A human-readable log of the processing steps applied.
|
||||
"""
|
||||
mutable struct MSISession
|
||||
session_id::String
|
||||
original_data::VersionedSpectralData
|
||||
history::Vector{VersionedSpectralData}
|
||||
current_version::VersionedSpectralData
|
||||
processing_steps::Vector{String}
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# Session Management
|
||||
# =============================================================================
|
||||
|
||||
"""
|
||||
initialize_session(raw_spectra::Vector, session_name::String) -> MSISession
|
||||
|
||||
Creates a new preprocessing session from raw spectral data.
|
||||
|
||||
# Arguments
|
||||
- `raw_spectra::Vector`: A vector of `(mz, intensity)` tuples representing the initial dataset.
|
||||
- `session_name::String`: A name for the session.
|
||||
|
||||
# Returns
|
||||
- `MSISession`: A new session object initialized with the provided data.
|
||||
"""
|
||||
function initialize_session(raw_spectra::Vector, session_name::String)::MSISession
|
||||
original = VersionedSpectralData(
|
||||
string(uuid4()),
|
||||
"root",
|
||||
raw_spectra,
|
||||
"Original Data",
|
||||
now(),
|
||||
Dict()
|
||||
)
|
||||
return MSISession(session_name, original, [original], original, [])
|
||||
end
|
||||
|
||||
"""
|
||||
get_processing_history(session::MSISession) -> Vector{String}
|
||||
|
||||
Returns a list of the processing steps applied during the session.
|
||||
"""
|
||||
get_processing_history(session::MSISession) = [v.processing_step for v in session.history]
|
||||
|
||||
"""
|
||||
undo_last_step!(session::MSISession)
|
||||
|
||||
Reverts the session to the state before the last processing step was applied.
|
||||
This modifies the session in place.
|
||||
"""
|
||||
function undo_last_step!(session::MSISession)
|
||||
if length(session.history) > 1
|
||||
pop!(session.history)
|
||||
pop!(session.processing_steps)
|
||||
session.current_version = last(session.history)
|
||||
end
|
||||
return session
|
||||
end
|
||||
|
||||
"""
|
||||
revert_to_version!(session::MSISession, version_id::String)
|
||||
|
||||
Reverts the session to a specific version in its history, discarding all
|
||||
subsequent changes. This modifies the session in place.
|
||||
"""
|
||||
function revert_to_version!(session::MSISession, version_id::String)
|
||||
target_idx = findfirst(v -> v.version_id == version_id, session.history)
|
||||
if !isnothing(target_idx)
|
||||
session.history = session.history[1:target_idx]
|
||||
session.current_version = session.history[end]
|
||||
# Also truncate the descriptive processing steps
|
||||
num_steps_to_keep = max(0, target_idx - 1)
|
||||
session.processing_steps = session.processing_steps[1:num_steps_to_keep]
|
||||
end
|
||||
return session
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# Versioned Preprocessing
|
||||
# =============================================================================
|
||||
|
||||
"""
|
||||
apply_processing_step(spectra::Vector, step::Symbol, params::Dict) -> Vector
|
||||
|
||||
A dispatcher that applies a single, specified preprocessing function to a set of spectra.
|
||||
This is the core function called by the versioned workflow.
|
||||
|
||||
# Arguments
|
||||
- `spectra::Vector`: The input spectral data.
|
||||
- `step::Symbol`: The symbol representing the processing step (e.g., `:smooth`, `:baseline`).
|
||||
- `params::Dict`: A dictionary of parameters for the step.
|
||||
|
||||
# Returns
|
||||
- `Vector`: The processed spectral data.
|
||||
"""
|
||||
function apply_processing_step(spectra::Vector, step::Symbol, params::Dict)
|
||||
processed = deepcopy(spectra)
|
||||
params_sym = Dict(Symbol(k) => v for (k,v) in params) # Ensure keys are symbols
|
||||
|
||||
if step === :transform
|
||||
for i in eachindex(processed)
|
||||
mz, y = processed[i]
|
||||
processed[i] = (mz, transform_intensity(y; params_sym...))
|
||||
end
|
||||
elseif step === :smooth
|
||||
for i in eachindex(processed)
|
||||
mz, y = processed[i]
|
||||
processed[i] = (mz, smooth_spectrum(y; params_sym...))
|
||||
end
|
||||
elseif step === :baseline
|
||||
for i in eachindex(processed)
|
||||
mz, y = processed[i]
|
||||
baseline = snip_baseline(y; params_sym...)
|
||||
processed[i] = (mz, max.(0.0, y .- baseline))
|
||||
end
|
||||
elseif step === :normalize
|
||||
mode = get(params, :normalize_method, :tic)
|
||||
if mode === :pqn
|
||||
matrix = hcat([float.(p[2]) for p in processed]...)
|
||||
matrix_norm = pqn_normalize(matrix)
|
||||
for i in eachindex(processed)
|
||||
mz, _ = processed[i]
|
||||
processed[i] = (mz, view(matrix_norm, :, i))
|
||||
end
|
||||
else # :tic or :median
|
||||
norm_func = (mode === :tic) ? tic_normalize : median_normalize
|
||||
for i in eachindex(processed)
|
||||
mz, y = processed[i]
|
||||
processed[i] = (mz, norm_func(y))
|
||||
end
|
||||
end
|
||||
elseif step === :peaks
|
||||
peak_results = Vector{Tuple{Vector{Float64},Vector{Float64}}}(undef, length(processed))
|
||||
for (i, (mz, y)) in enumerate(processed)
|
||||
pk_mz, pk_int = detect_peaks_profile(mz, y; params_sym...)
|
||||
peak_results[i] = (pk_mz, pk_int)
|
||||
end
|
||||
return peak_results
|
||||
elseif step === :peaks_wavelet
|
||||
peak_results = Vector{Tuple{Vector{Float64},Vector{Float64}}}(undef, length(processed))
|
||||
for (i, (mz, y)) in enumerate(processed)
|
||||
pk_mz, pk_int = detect_peaks_wavelet(mz, y; params_sym...)
|
||||
peak_results[i] = (pk_mz, pk_int)
|
||||
end
|
||||
return peak_results
|
||||
elseif step === :calibrate
|
||||
return calibrate_spectra(processed; params_sym...)
|
||||
else
|
||||
@warn "Unsupported processing step: $step"
|
||||
end
|
||||
return processed
|
||||
end
|
||||
|
||||
"""
|
||||
run_versioned_preprocessing(session::MSISession, step::Symbol, params::Dict) -> VersionedSpectralData
|
||||
|
||||
Creates a new version of spectral data by applying a single processing step to the
|
||||
current version in the session.
|
||||
|
||||
# Arguments
|
||||
- `session::MSISession`: The current processing session.
|
||||
- `step::Symbol`: The processing step to apply.
|
||||
- `params::Dict`: Parameters for the processing step.
|
||||
|
||||
# Returns
|
||||
- `VersionedSpectralData`: A new data version with the transformation applied.
|
||||
"""
|
||||
function run_versioned_preprocessing(session::MSISession, step::Symbol, params::Dict)
|
||||
# Create a new version based on the current one
|
||||
parent_version = session.current_version
|
||||
new_version_id = string(uuid4())
|
||||
|
||||
# Apply the processing step
|
||||
new_spectra = apply_processing_step(parent_version.spectra, step, params)
|
||||
|
||||
# Create the new versioned data object
|
||||
new_version = VersionedSpectralData(
|
||||
new_version_id,
|
||||
parent_version.version_id,
|
||||
new_spectra,
|
||||
string(step),
|
||||
now(),
|
||||
params
|
||||
)
|
||||
|
||||
return new_version
|
||||
end
|
||||
|
||||
"""
|
||||
apply_processing_with_version!(session::MSISession, step::Symbol, ui_params::Dict)
|
||||
|
||||
A high-level function to apply a processing step, create a new version, and
|
||||
update the session state in place.
|
||||
|
||||
# Arguments
|
||||
- `session::MSISession`: The session to modify.
|
||||
- `step::Symbol`: The processing step to apply.
|
||||
- `ui_params::Dict`: A dictionary of parameters, typically from a UI.
|
||||
"""
|
||||
function apply_processing_with_version!(session::MSISession, step::Symbol, ui_params::Dict)
|
||||
# In a real app, you would validate and convert UI params here.
|
||||
processing_params = ui_params
|
||||
|
||||
# Create the new version
|
||||
new_version = run_versioned_preprocessing(session, step, processing_params)
|
||||
|
||||
# Update the session
|
||||
push!(session.history, new_version)
|
||||
session.current_version = new_version
|
||||
|
||||
# Describe the step for the history log
|
||||
step_description = "$(string(step)) with params: " * join(["$k=$v" for (k,v) in processing_params], ", ")
|
||||
push!(session.processing_steps, step_description)
|
||||
|
||||
return session
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# Data Serialization & Export
|
||||
# =============================================================================
|
||||
|
||||
"""
|
||||
save_spectral_version(data::VersionedSpectralData, filepath::String)
|
||||
|
||||
Saves a `VersionedSpectralData` object to a file using the JLD2 format.
|
||||
"""
|
||||
function save_spectral_version(data::VersionedSpectralData, filepath::String)
|
||||
JLD2.save_object(filepath, data)
|
||||
end
|
||||
|
||||
"""
|
||||
load_spectral_version(filepath::String) -> VersionedSpectralData
|
||||
|
||||
Loads a `VersionedSpectralData` object from a JLD2 file.
|
||||
"""
|
||||
function load_spectral_version(filepath::String)::VersionedSpectralData
|
||||
return JLD2.load_object(filepath)
|
||||
end
|
||||
|
||||
"""
|
||||
_encode_base64(data::AbstractVector{<:Real}) -> String
|
||||
|
||||
Helper function to convert a numeric vector into a Base64 encoded string.
|
||||
"""
|
||||
function _encode_base64(data::AbstractVector{T}) where T <: Real
|
||||
bytes = reinterpret(UInt8, data)
|
||||
return base64encode(bytes)
|
||||
end
|
||||
|
||||
"""
|
||||
export_to_mzml(session::MSISession, filepath::String)
|
||||
|
||||
Exports the current version of the spectral data in a session to a standard
|
||||
.mzML file.
|
||||
|
||||
# Arguments
|
||||
- `session::MSISession`: The current session.
|
||||
- `filepath::String`: The path for the output .mzML file.
|
||||
"""
|
||||
function export_to_mzml(session::MSISession, filepath::String)
|
||||
spectra_to_export = session.current_version.spectra
|
||||
|
||||
open(filepath, "w") do f
|
||||
# XML Header
|
||||
write(f, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
|
||||
write(f, "<mzML xmlns=\"http://psi.hupo.org/ms/mzml\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://psi.hupo.org/ms/mzml http://psidev.info/files/ms/mzML/xsd/mzML1.1.0.xsd\" version=\"1.1\">\n")
|
||||
|
||||
# CV List
|
||||
write(f, " <cvList count=\"2\">\n")
|
||||
write(f, " <cv id=\"MS\" fullName=\"Proteomics Standards Initiative Mass Spectrometry Ontology\" version=\"4.1.123\" URI=\"https://raw.githubusercontent.com/HUPO-PSI/psi-ms-CV/master/psi-ms.obo\"/>\n")
|
||||
write(f, " <cv id=\"UO\" fullName=\"Unit Ontology\" version=\"07:03:2022\" URI=\"https://raw.githubusercontent.com/bio-ontology-research-group/unit-ontology/master/unit.obo\"/>\n")
|
||||
write(f, " </cvList>\n")
|
||||
|
||||
# File Description
|
||||
write(f, " <fileDescription>\n <fileContent>\n")
|
||||
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000579\" name=\"MS1 spectrum\"/>\n")
|
||||
write(f, " </fileContent>\n </fileDescription>\n")
|
||||
|
||||
# Run and Spectrum List
|
||||
write(f, " <run id=\"run1\" defaultInstrumentConfigurationRef=\"instrument1\">\n")
|
||||
write(f, " <spectrumList count=\"$(length(spectra_to_export))\" defaultDataProcessingRef=\"dp1\">\n")
|
||||
|
||||
for (i, (mz, intensity)) in enumerate(spectra_to_export)
|
||||
# Ensure data is in the correct format for encoding
|
||||
mz_64 = convert(Vector{Float64}, mz)
|
||||
int_32 = convert(Vector{Float32}, intensity)
|
||||
|
||||
# Base64 encode the binary data
|
||||
mz_b64 = _encode_base64(mz_64)
|
||||
int_b64 = _encode_base64(int_32)
|
||||
|
||||
write(f, " <spectrum index=\"$(i-1)\" id=\"scan=$(i)\" defaultArrayLength=\"$(length(mz))\">\n")
|
||||
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000511\" name=\"ms level\" value=\"1\"/>\n")
|
||||
# Assuming profile mode for processed data, could be made dynamic
|
||||
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000128\" name=\"profile spectrum\"/>\n")
|
||||
|
||||
write(f, " <binaryDataArrayList count=\"2\">\n")
|
||||
|
||||
# m/z array
|
||||
write(f, " <binaryDataArray encodedLength=\"$(length(mz_b64))\">\n")
|
||||
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000514\" name=\"m/z array\" unitCvRef=\"MS\" unitAccession=\"MS:1000040\" unitName=\"m/z\"/>\n")
|
||||
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000523\" name=\"64-bit float\"/>\n")
|
||||
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000576\" name=\"no compression\"/>\n")
|
||||
write(f, " <binary>$(mz_b64)</binary>\n")
|
||||
write(f, " </binaryDataArray>\n")
|
||||
|
||||
# Intensity array
|
||||
write(f, " <binaryDataArray encodedLength=\"$(length(int_b64))\">\n")
|
||||
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000515\" name=\"intensity array\" unitCvRef=\"MS\" unitAccession=\"MS:1000131\" unitName=\"number of detector counts\"/>\n")
|
||||
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000521\" name=\"32-bit float\"/>\n")
|
||||
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000576\" name=\"no compression\"/>\n")
|
||||
write(f, " <binary>$(int_b64)</binary>\n")
|
||||
write(f, " </binaryDataArray>\n")
|
||||
|
||||
write(f, " </binaryDataArrayList>\n")
|
||||
write(f, " </spectrum>\n")
|
||||
end
|
||||
|
||||
write(f, " </spectrumList>\n")
|
||||
write(f, " </run>\n")
|
||||
|
||||
# Dummy instrument and data processing info
|
||||
write(f, " <instrumentConfigurationList count=\"1\">\n")
|
||||
write(f, " <instrumentConfiguration id=\"instrument1\"/>\n")
|
||||
write(f, " </instrumentConfigurationList>\n")
|
||||
write(f, " <dataProcessingList count=\"1\">\n")
|
||||
write(f, " <dataProcessing id=\"dp1\"/>\n")
|
||||
write(f, " </dataProcessingList>\n")
|
||||
|
||||
write(f, "</mzML>\n")
|
||||
end
|
||||
println("Successfully exported current data to $filepath")
|
||||
end
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Stubs for UI Integration
|
||||
# =============================================================================
|
||||
|
||||
"""
|
||||
validate_ui_parameters(step::Symbol, ui_params::Dict) -> Tuple{Bool, String}
|
||||
|
||||
(Stub) Validates parameters from a UI before they are used in a processing step.
|
||||
"""
|
||||
function validate_ui_parameters(step::Symbol, ui_params::Dict)::Tuple{Bool, String}
|
||||
# In a real implementation, this would check types, ranges, etc.
|
||||
# For example, for :smooth, ensure 'sg_window' is an odd integer.
|
||||
println("Validating parameters for step: $step")
|
||||
return (true, "Parameters are valid.")
|
||||
end
|
||||
|
||||
"""
|
||||
run_processing_with_progress(session, step, params, progress_callback)
|
||||
|
||||
(Stub) A wrapper for running a processing step that includes a progress reporting callback.
|
||||
"""
|
||||
function run_processing_with_progress(session, step, params, progress_callback)
|
||||
progress_callback(0.0, "Starting $step...")
|
||||
|
||||
# This is a simplified example. Real implementation would need to
|
||||
# hook into the loops inside apply_processing_step.
|
||||
new_version = run_versioned_preprocessing(session, step, params)
|
||||
|
||||
progress_callback(1.0, "Finished $step.")
|
||||
return new_version
|
||||
end
|
||||
|
||||
"""
|
||||
safe_processing_application!(session::MSISession, step::Symbol, params::Dict)
|
||||
|
||||
(Stub) A safe wrapper to apply a processing step that includes error handling and rollback.
|
||||
"""
|
||||
function safe_processing_application!(session::MSISession, step::Symbol, params::Dict)
|
||||
num_history = length(session.history)
|
||||
try
|
||||
println("Safely applying step: $step")
|
||||
apply_processing_with_version!(session, step, params)
|
||||
catch e
|
||||
@error "Processing step '$step' failed!" exception=(e, catch_backtrace())
|
||||
# Roll back to the previous state
|
||||
while length(session.history) > num_history
|
||||
undo_last_step!(session)
|
||||
end
|
||||
println("Session has been rolled back to the previous state.")
|
||||
return session # Return the rolled-back session
|
||||
end
|
||||
return session # Return the updated session
|
||||
end
|
||||
|
||||
320
src/imzML.jl
320
src/imzML.jl
@ -17,6 +17,134 @@ Core Functions:
|
||||
#
|
||||
# ============================================================================
|
||||
|
||||
function parse_imzml_header(stream::IO)
|
||||
# Initialize all return values and temporary variables
|
||||
instrument_meta = InstrumentMetadata()
|
||||
param_groups = Dict{String, SpecDim}()
|
||||
img_dims = zeros(Int32, 3)
|
||||
vendor_preprocessing_steps = String[] # Initialize vendor_preprocessing_steps
|
||||
|
||||
# Temp vars for parsing
|
||||
resolution = instrument_meta.resolution
|
||||
instrument_model = instrument_meta.instrument_model
|
||||
mass_accuracy_ppm = instrument_meta.mass_accuracy_ppm
|
||||
polarity = instrument_meta.polarity
|
||||
calibration_status = instrument_meta.calibration_status
|
||||
laser_settings = Dict{String, Any}()
|
||||
|
||||
try
|
||||
seekstart(stream)
|
||||
|
||||
in_scan_settings = false
|
||||
dim_axes_found = 0
|
||||
found_spectrum_count = false
|
||||
|
||||
# This loop performs a single pass to get most metadata
|
||||
while !eof(stream)
|
||||
line = readline(stream)
|
||||
|
||||
# State-machine like logic to enter/exit sections
|
||||
if occursin("<scanSettings", line)
|
||||
in_scan_settings = true
|
||||
elseif occursin("</scanSettings>", line)
|
||||
in_scan_settings = false
|
||||
end
|
||||
|
||||
# --- Parse Scan Settings for Image Dimensions ---
|
||||
if in_scan_settings && occursin("<cvParam", line) && dim_axes_found < 2
|
||||
accession_match = get_attribute(line, "accession")
|
||||
if accession_match !== nothing
|
||||
accession = accession_match.captures[1]
|
||||
if accession == "IMS:1000042" || accession == "IMS:1000043" # max dimension x or y
|
||||
value_match = get_attribute(line, "value")
|
||||
if value_match !== nothing
|
||||
axis_idx = (accession == "IMS:1000042") ? 1 : 2
|
||||
img_dims[axis_idx] = parse(Int32, value_match.captures[1])
|
||||
dim_axes_found += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# --- Parse Spectrum List for total count ---
|
||||
if occursin("<spectrumList", line)
|
||||
count_match = get_attribute(line, "count")
|
||||
if count_match !== nothing
|
||||
img_dims[3] = parse(Int32, count_match.captures[1])
|
||||
found_spectrum_count = true
|
||||
end
|
||||
end
|
||||
|
||||
# --- Parse general CV params for InstrumentMetadata ---
|
||||
if occursin("<cvParam", line)
|
||||
accession_match = get_attribute(line, "accession")
|
||||
value_match = get_attribute(line, "value")
|
||||
name_match = get_attribute(line, "name") # For laser attributes and vendor_preprocessing names
|
||||
|
||||
if accession_match !== nothing
|
||||
acc = accession_match.captures[1]
|
||||
val = (value_match !== nothing) ? value_match.captures[1] : ""
|
||||
cv_name = (name_match !== nothing) ? name_match.captures[1] : "" # Use cv_name to avoid conflict
|
||||
|
||||
if acc == "MS:1000031" # instrument model
|
||||
instrument_model = val
|
||||
elseif acc == "MS:1001496" # mass resolving power (more specific)
|
||||
resolution = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000011" && resolution === nothing # resolution (less specific)
|
||||
resolution = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000016" # mass accuracy (ppm)
|
||||
mass_accuracy_ppm = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000130" # positive scan
|
||||
polarity = :positive
|
||||
elseif acc == "MS:1000129" # negative scan
|
||||
polarity = :negative
|
||||
elseif acc == "MS:1000592" # external calibration
|
||||
calibration_status = :external
|
||||
elseif acc == "MS:1000593" # internal calibration
|
||||
calibration_status = :internal
|
||||
elseif acc == "MS:1000747" && calibration_status == :uncalibrated # instrument specific calibration
|
||||
calibration_status = :internal # Assume as a form of internal calibration
|
||||
elseif acc == "MS:1000867" # laser wavelength
|
||||
laser_settings["wavelength_nm"] = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000868" # laser fluence
|
||||
laser_settings["fluence"] = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000869" # laser repetition rate
|
||||
laser_settings["repetition_rate_hz"] = tryparse(Float64, val)
|
||||
# NEW: Vendor Preprocessing terms
|
||||
elseif acc == "MS:1000579" # baseline correction
|
||||
push!(vendor_preprocessing_steps, "Baseline Correction")
|
||||
elseif acc == "MS:1000580" # smoothing
|
||||
push!(vendor_preprocessing_steps, "Smoothing")
|
||||
elseif acc == "MS:1000578" # data transformation (e.g., centroiding)
|
||||
push!(vendor_preprocessing_steps, "Data Transformation: $(cv_name)") # Use cv_name here
|
||||
elseif acc == "MS:1000800" # deisotoping
|
||||
push!(vendor_preprocessing_steps, "Deisotoping")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if found_spectrum_count && dim_axes_found == 2
|
||||
break
|
||||
end
|
||||
end
|
||||
catch e
|
||||
@warn "Could not fully parse imzML header in single pass. Error: $e"
|
||||
end
|
||||
|
||||
# The axes_config_img logic is complex to merge; we run it as a second, small pass.
|
||||
seekstart(stream)
|
||||
param_groups = axes_config_img(stream)
|
||||
|
||||
final_instrument_meta = InstrumentMetadata(
|
||||
resolution, :unknown, :unknown, calibration_status,
|
||||
instrument_model, mass_accuracy_ppm,
|
||||
isempty(laser_settings) ? nothing : laser_settings,
|
||||
polarity,
|
||||
isempty(vendor_preprocessing_steps) ? nothing : vendor_preprocessing_steps # NEW field
|
||||
)
|
||||
return (final_instrument_meta, param_groups, img_dims)
|
||||
end
|
||||
|
||||
"""
|
||||
axes_config_img(stream)
|
||||
|
||||
@ -44,40 +172,6 @@ function axes_config_img(stream::IO)
|
||||
return param_groups
|
||||
end
|
||||
|
||||
"""
|
||||
get_img_dimensions(stream)
|
||||
|
||||
Reads the maximum X and Y dimensions and total spectrum count from the metadata.
|
||||
"""
|
||||
function get_img_dimensions(stream::IO)
|
||||
find_tag(stream, r"^\s*<(scanSettings )")
|
||||
n = 2
|
||||
dim = zeros(Int32, 3)
|
||||
|
||||
while n > 0 && !eof(stream)
|
||||
currLine = readline(stream)
|
||||
if occursin("<cvParam", currLine)
|
||||
accession_match = get_attribute(currLine, "accession")
|
||||
if accession_match !== nothing
|
||||
accession = accession_match.captures[1]
|
||||
if accession == "IMS:1000042" || accession == "IMS:1000043"
|
||||
value_match = get_attribute(currLine, "value")
|
||||
if value_match !== nothing
|
||||
axis_idx = (accession == "IMS:1000042") ? 1 : 2
|
||||
dim[axis_idx] = parse(Int32, value_match.captures[1])
|
||||
n -= 1
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
count_tag = find_tag(stream, r"^\s*<spectrumList(.+)")
|
||||
count_match = get_attribute(count_tag.captures[1], "count")
|
||||
dim[3] = parse(Int32, count_match.captures[1])
|
||||
return dim
|
||||
end
|
||||
|
||||
"""
|
||||
get_spectrum_tag_offset(stream)
|
||||
|
||||
@ -179,13 +273,9 @@ function determine_parser(stream::IO, mz_is_compressed::Bool, int_is_compressed:
|
||||
return :neofx
|
||||
end
|
||||
|
||||
if mz_is_compressed || int_is_compressed || has_external_data_markers
|
||||
# If not neofx, it must be compressed (as uncompressed path is no longer supported)
|
||||
return :compressed
|
||||
end
|
||||
|
||||
return :uncompressed
|
||||
end
|
||||
|
||||
function load_imzml_lazy(file_path::String; cache_size::Int=100)
|
||||
println("DEBUG: Checking for .imzML file at $file_path")
|
||||
if !isfile(file_path)
|
||||
@ -203,10 +293,21 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
|
||||
ts_hIbd = ThreadSafeFileHandle(ibd_path)
|
||||
|
||||
try
|
||||
println("DEBUG: Configuring axes...")
|
||||
param_groups = axes_config_img(stream)
|
||||
println("DEBUG: Getting image dimensions...")
|
||||
imgDim = get_img_dimensions(stream)
|
||||
# --- NEW: Parse all header information in a more efficient single pass ---
|
||||
println("DEBUG: Parsing imzML header...")
|
||||
(instrument_meta, param_groups, imgDim) = parse_imzml_header(stream)
|
||||
# The header parser will have reset the stream for the next step (spectrum parsing)
|
||||
|
||||
println("--- Extracted Instrument Metadata ---")
|
||||
println("Resolution: ", instrument_meta.resolution)
|
||||
println("Acquisition Mode (pre-check): ", instrument_meta.acquisition_mode)
|
||||
println("Calibration Status: ", instrument_meta.calibration_status)
|
||||
println("Instrument Model: ", instrument_meta.instrument_model)
|
||||
println("Mass Accuracy (ppm): ", instrument_meta.mass_accuracy_ppm)
|
||||
println("Laser Settings: ", instrument_meta.laser_settings)
|
||||
println("Polarity: ", instrument_meta.polarity)
|
||||
println("------------------------------------")
|
||||
|
||||
width, height, num_spectra = imgDim
|
||||
println("DEBUG: Image dimensions: $(width)x$(height), $num_spectra spectra.")
|
||||
|
||||
@ -251,19 +352,6 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
|
||||
spectra_metadata = parse_neofx(stream, ts_hIbd, param_groups, width, height, num_spectra,
|
||||
default_mz_format, default_intensity_format,
|
||||
mz_is_compressed, int_is_compressed, global_mode)
|
||||
#=
|
||||
elseif parser_type == :compressed
|
||||
println("DEBUG: Using compressed parser.")
|
||||
spectra_metadata = parse_compressed(stream, hIbd, param_groups, width, height, num_spectra,
|
||||
default_mz_format, default_intensity_format,
|
||||
mz_is_compressed, int_is_compressed, global_mode)
|
||||
else # :uncompressed
|
||||
println("DEBUG: Using uncompressed parser.")
|
||||
spectra_metadata = parse_uncompressed(stream, hIbd, param_groups, width, height, num_spectra,
|
||||
default_mz_format, default_intensity_format,
|
||||
mz_is_compressed, int_is_compressed, global_mode)
|
||||
end
|
||||
=#
|
||||
else
|
||||
println("DEBUG: Using compressed parser.")
|
||||
spectra_metadata = parse_compressed(stream, ts_hIbd, param_groups, width, height, num_spectra,
|
||||
@ -286,9 +374,30 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
|
||||
end
|
||||
println("DEBUG: Coordinate map built.")
|
||||
|
||||
# --- NEW: Update acquisition mode based on spectrum parsing ---
|
||||
acq_mode_symbol = if global_mode == CENTROID
|
||||
:centroid
|
||||
elseif global_mode == PROFILE
|
||||
:profile
|
||||
else
|
||||
:unknown
|
||||
end
|
||||
|
||||
final_instrument_meta = InstrumentMetadata(
|
||||
instrument_meta.resolution,
|
||||
acq_mode_symbol, # Update with parsed mode
|
||||
instrument_meta.mz_axis_type,
|
||||
instrument_meta.calibration_status,
|
||||
instrument_meta.instrument_model,
|
||||
instrument_meta.mass_accuracy_ppm,
|
||||
instrument_meta.laser_settings,
|
||||
instrument_meta.polarity,
|
||||
instrument_meta.vendor_preprocessing_steps # Add this new field
|
||||
)
|
||||
|
||||
source = ImzMLSource(ts_hIbd, default_mz_format, default_intensity_format)
|
||||
println("DEBUG: Creating MSIData object.")
|
||||
msi_data = MSIData(source, spectra_metadata, (width, height), coordinate_map, cache_size)
|
||||
msi_data = MSIData(source, spectra_metadata, final_instrument_meta, (width, height), coordinate_map, cache_size)
|
||||
|
||||
# Close the XML stream as it's no longer needed
|
||||
close(stream)
|
||||
@ -302,99 +411,6 @@ function load_imzml_lazy(file_path::String; cache_size::Int=100)
|
||||
end
|
||||
end
|
||||
|
||||
#=
|
||||
function parse_uncompressed(stream::IO, hIbd::IO, param_groups::Dict{String, SpecDim},
|
||||
width::Int32, height::Int32, num_spectra::Int32,
|
||||
mz_format::DataType, intensity_format::DataType,
|
||||
mz_is_compressed::Bool, int_is_compressed::Bool, global_mode::SpectrumMode)
|
||||
|
||||
println("DEBUG: Learning file structure from first spectrum...")
|
||||
start_of_spectra_xml = position(stream)
|
||||
|
||||
# Skip UUID at beginning of IBD file (from converter)
|
||||
seek(hIbd, 16)
|
||||
current_ibd_offset = position(hIbd)
|
||||
|
||||
attr = get_spectrum_attributes(stream, hIbd)
|
||||
seek(stream, start_of_spectra_xml)
|
||||
println("DEBUG: Initial IBD offset (after UUID): $current_ibd_offset")
|
||||
|
||||
spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra)
|
||||
mz_is_first = attr[3] == 3
|
||||
|
||||
for k in 1:num_spectra
|
||||
spectrum_start_pos = position(stream)
|
||||
|
||||
# Find X coordinate
|
||||
line = readuntil(stream, "IMS:1000050")
|
||||
if eof(stream)
|
||||
throw(FileFormatError("Unexpected EOF while looking for X coordinate"))
|
||||
end
|
||||
val_tag_x = find_tag(stream, r"value=\"(\d+)\"")
|
||||
x = parse(Int32, val_tag_x.captures[1])
|
||||
|
||||
# Find Y coordinate
|
||||
line = readuntil(stream, "IMS:1000051")
|
||||
val_tag_y = find_tag(stream, r"value=\"(\d+)\"")
|
||||
y = parse(Int32, val_tag_y.captures[1])
|
||||
|
||||
# Find array length - look for external array length
|
||||
line = readuntil(stream, "IMS:1000103")
|
||||
val_tag_len = find_tag(stream, r"value=\"(\d+)\"")
|
||||
nPoints = parse(Int32, val_tag_len.captures[1])
|
||||
|
||||
# CRITICAL FIX: Use the actual formats from the XML, not hardcoded values
|
||||
# The converter writes Float64 for mz, Float32 for intensity
|
||||
mz_len_bytes = nPoints * sizeof(Float64) # Converter uses Float64 for mz
|
||||
int_len_bytes = nPoints * sizeof(Float32) # Converter uses Float32 for intensity
|
||||
|
||||
local mz_offset, int_offset
|
||||
if mz_is_first
|
||||
mz_offset = current_ibd_offset
|
||||
int_offset = mz_offset + mz_len_bytes
|
||||
else
|
||||
int_offset = current_ibd_offset
|
||||
mz_offset = int_offset + int_len_bytes
|
||||
end
|
||||
|
||||
# Mode detection
|
||||
current_pos = position(stream)
|
||||
seek(stream, spectrum_start_pos)
|
||||
spectrum_buffer = IOBuffer()
|
||||
line = ""
|
||||
while !eof(stream)
|
||||
line = readline(stream)
|
||||
write(spectrum_buffer, line)
|
||||
if occursin("</spectrum>", line)
|
||||
break
|
||||
end
|
||||
end
|
||||
spectrum_xml = String(take!(spectrum_buffer))
|
||||
|
||||
spectrum_mode = global_mode
|
||||
if occursin("MS:1000127", spectrum_xml)
|
||||
spectrum_mode = CENTROID
|
||||
elseif occursin("MS:1000128", spectrum_xml)
|
||||
spectrum_mode = PROFILE
|
||||
end
|
||||
seek(stream, current_pos)
|
||||
|
||||
# CRITICAL FIX: Use correct data types that match the converter output
|
||||
mz_asset = SpectrumAsset(Float64, mz_is_compressed, mz_offset, nPoints, :mz)
|
||||
int_asset = SpectrumAsset(Float32, int_is_compressed, int_offset, nPoints, :intensity)
|
||||
|
||||
spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset)
|
||||
|
||||
current_ibd_offset += mz_len_bytes + int_len_bytes
|
||||
|
||||
# Skip to next spectrum
|
||||
line = readuntil(stream, "</spectrum>")
|
||||
end
|
||||
|
||||
return spectra_metadata
|
||||
end
|
||||
=#
|
||||
|
||||
function parse_compressed(stream::IO, hIbd::Union{IO, ThreadSafeFileHandle}, param_groups::Dict{String, SpecDim},
|
||||
width::Int32, height::Int32, num_spectra::Int32,
|
||||
default_mz_format::DataType, default_intensity_format::DataType,
|
||||
@ -532,7 +548,7 @@ function parse_compressed(stream::IO, hIbd::Union{IO, ThreadSafeFileHandle}, par
|
||||
int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity)
|
||||
end
|
||||
|
||||
spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset)
|
||||
spectra_metadata[k] = SpectrumMetadata(x, y, "", :sample, spectrum_mode, mz_asset, int_asset)
|
||||
end
|
||||
|
||||
return spectra_metadata
|
||||
@ -675,7 +691,7 @@ function parse_neofx(stream::IO, hIbd::Union{IO, ThreadSafeFileHandle}, param_gr
|
||||
int_is_compressed ? int_info.encoded_length : int_info.array_length, :intensity)
|
||||
end
|
||||
|
||||
spectra_metadata[k] = SpectrumMetadata(x, y, "", spectrum_mode, mz_asset, int_asset)
|
||||
spectra_metadata[k] = SpectrumMetadata(x, y, "", :sample, spectrum_mode, mz_asset, int_asset)
|
||||
end
|
||||
|
||||
return spectra_metadata
|
||||
|
||||
148
src/mzML.jl
148
src/mzML.jl
@ -19,6 +19,107 @@ const DATA_FORMAT_ACCESSIONS = Dict{String, DataType}(
|
||||
"MS:1000523" => Float64
|
||||
)
|
||||
|
||||
function parse_instrument_metadata_mzml(stream::IO)
|
||||
# Initialize with default values from the InstrumentMetadata constructor
|
||||
instrument_meta = InstrumentMetadata()
|
||||
|
||||
# Create temporary variables to hold parsed values
|
||||
resolution = instrument_meta.resolution
|
||||
instrument_model = instrument_meta.instrument_model
|
||||
mass_accuracy_ppm = instrument_meta.mass_accuracy_ppm
|
||||
polarity = instrument_meta.polarity
|
||||
calibration_status = instrument_meta.calibration_status
|
||||
laser_settings = Dict{String, Any}() # Use Any for heterogeneous values
|
||||
vendor_preprocessing_steps = String[] # Initialize vendor_preprocessing_steps
|
||||
|
||||
try
|
||||
# Reset stream and read a sufficiently large header block to find metadata.
|
||||
seekstart(stream)
|
||||
header_block = ""
|
||||
header_read_limit = 50000 # Read up to 50KB of header
|
||||
bytes_read = 0
|
||||
|
||||
while !eof(stream) && bytes_read < header_read_limit
|
||||
line_pos = position(stream)
|
||||
line = readline(stream)
|
||||
bytes_read += (position(stream) - line_pos)
|
||||
|
||||
# Stop at the start of the main data section
|
||||
if occursin("<run>", line) || occursin("<spectrumList>", line)
|
||||
break
|
||||
end
|
||||
header_block *= line * "\n"
|
||||
end
|
||||
|
||||
header_io = IOBuffer(header_block)
|
||||
while !eof(header_io)
|
||||
line = readline(header_io)
|
||||
|
||||
if occursin("<cvParam", line)
|
||||
accession_match = get_attribute(line, "accession")
|
||||
value_match = get_attribute(line, "value")
|
||||
name_match = get_attribute(line, "name") # For laser attributes and vendor_preprocessing names
|
||||
|
||||
if accession_match !== nothing
|
||||
acc = accession_match.captures[1]
|
||||
val = (value_match !== nothing) ? value_match.captures[1] : ""
|
||||
name = (name_match !== nothing) ? name_match.captures[1] : "" # NEW: Get name for logging
|
||||
|
||||
if acc == "MS:1000031" # instrument model
|
||||
instrument_model = val
|
||||
elseif acc == "MS:1001496" # mass resolving power (more specific)
|
||||
resolution = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000011" && resolution === nothing # resolution (less specific)
|
||||
resolution = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000016" # mass accuracy (ppm)
|
||||
mass_accuracy_ppm = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000130" # positive scan
|
||||
polarity = :positive
|
||||
elseif acc == "MS:1000129" # negative scan
|
||||
polarity = :negative
|
||||
elseif acc == "MS:1000592" # external calibration
|
||||
calibration_status = :external
|
||||
elseif acc == "MS:1000593" # internal calibration
|
||||
calibration_status = :internal
|
||||
elseif acc == "MS:1000747" && calibration_status == :uncalibrated # instrument specific calibration
|
||||
calibration_status = :internal # Assume as a form of internal calibration
|
||||
elseif acc == "MS:1000867" # laser wavelength
|
||||
laser_settings["wavelength_nm"] = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000868" # laser fluence
|
||||
laser_settings["fluence"] = tryparse(Float64, val)
|
||||
elseif acc == "MS:1000869" # laser repetition rate
|
||||
laser_settings["repetition_rate_hz"] = tryparse(Float64, val)
|
||||
# NEW: Vendor Preprocessing terms
|
||||
elseif acc == "MS:1000579" # baseline correction
|
||||
push!(vendor_preprocessing_steps, "Baseline Correction")
|
||||
elseif acc == "MS:1000580" # smoothing
|
||||
push!(vendor_preprocessing_steps, "Smoothing")
|
||||
elseif acc == "MS:1000578" # data transformation (e.g., centroiding)
|
||||
push!(vendor_preprocessing_steps, "Data Transformation: $(name)")
|
||||
elseif acc == "MS:1000800" # deisotoping
|
||||
push!(vendor_preprocessing_steps, "Deisotoping")
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
catch e
|
||||
@warn "Could not fully parse instrument metadata from mzML header. Using defaults. Error: $e"
|
||||
end
|
||||
|
||||
# Always return a valid object
|
||||
return InstrumentMetadata(
|
||||
resolution,
|
||||
instrument_meta.acquisition_mode, # To be determined by load_mzml_lazy
|
||||
instrument_meta.mz_axis_type,
|
||||
calibration_status,
|
||||
instrument_model,
|
||||
mass_accuracy_ppm,
|
||||
isempty(laser_settings) ? nothing : laser_settings,
|
||||
polarity,
|
||||
isempty(vendor_preprocessing_steps) ? nothing : vendor_preprocessing_steps # NEW field
|
||||
)
|
||||
end
|
||||
|
||||
"""
|
||||
get_spectrum_asset_metadata(stream::IO)
|
||||
|
||||
@ -149,7 +250,7 @@ function parse_spectrum_metadata(stream::IO, offset::Int64)
|
||||
|
||||
# Create the new unified metadata object
|
||||
# For mzML, x and y coordinates are not applicable, so we use 0.
|
||||
return SpectrumMetadata(Int32(0), Int32(0), id, mode, mz_asset, int_asset)
|
||||
return SpectrumMetadata(Int32(0), Int32(0), id, :sample, mode, mz_asset, int_asset)
|
||||
end
|
||||
|
||||
"""
|
||||
@ -231,6 +332,22 @@ function load_mzml_lazy(file_path::String; cache_size::Int=100)
|
||||
ts_stream = ThreadSafeFileHandle(file_path, "r")
|
||||
|
||||
try
|
||||
# --- NEW: Parse instrument metadata from header ---
|
||||
println("DEBUG: Parsing instrument metadata from header...")
|
||||
instrument_meta = parse_instrument_metadata_mzml(ts_stream.handle)
|
||||
|
||||
println("--- Extracted Instrument Metadata ---")
|
||||
println("Resolution: ", instrument_meta.resolution)
|
||||
println("Acquisition Mode (pre-check): ", instrument_meta.acquisition_mode)
|
||||
println("Calibration Status: ", instrument_meta.calibration_status)
|
||||
println("Instrument Model: ", instrument_meta.instrument_model)
|
||||
println("Mass Accuracy (ppm): ", instrument_meta.mass_accuracy_ppm)
|
||||
println("Laser Settings: ", instrument_meta.laser_settings)
|
||||
println("Polarity: ", instrument_meta.polarity)
|
||||
println("------------------------------------")
|
||||
|
||||
seekstart(ts_stream.handle) # Reset stream after header parsing
|
||||
|
||||
println("DEBUG: Finding index offset...")
|
||||
index_offset = find_index_offset(ts_stream.handle)
|
||||
println("DEBUG: Seeking to index list at offset $index_offset.")
|
||||
@ -270,9 +387,36 @@ function load_mzml_lazy(file_path::String; cache_size::Int=100)
|
||||
mz_format = first_meta.mz_asset.format
|
||||
intensity_format = first_meta.int_asset.format
|
||||
|
||||
# --- NEW: Determine overall acquisition mode ---
|
||||
modes = [meta.mode for meta in spectra_metadata]
|
||||
num_centroid = count(m -> m == CENTROID, modes)
|
||||
num_profile = count(m -> m == PROFILE, modes)
|
||||
|
||||
acq_mode_symbol = if num_centroid > 0 && num_profile == 0
|
||||
:centroid
|
||||
elseif num_profile > 0 && num_centroid == 0
|
||||
:profile
|
||||
elseif num_centroid > 0 && num_profile > 0
|
||||
:mixed
|
||||
else
|
||||
:unknown
|
||||
end
|
||||
|
||||
final_instrument_meta = InstrumentMetadata(
|
||||
instrument_meta.resolution,
|
||||
acq_mode_symbol, # Update with parsed mode
|
||||
instrument_meta.mz_axis_type,
|
||||
instrument_meta.calibration_status,
|
||||
instrument_meta.instrument_model,
|
||||
instrument_meta.mass_accuracy_ppm,
|
||||
instrument_meta.laser_settings,
|
||||
instrument_meta.polarity,
|
||||
instrument_meta.vendor_preprocessing_steps # Add this new field
|
||||
)
|
||||
|
||||
source = MzMLSource(ts_stream, mz_format, intensity_format)
|
||||
println("DEBUG: Creating MSIData object.")
|
||||
return MSIData(source, spectra_metadata, (0, 0), nothing, cache_size)
|
||||
return MSIData(source, spectra_metadata, final_instrument_meta, (0, 0), nothing, cache_size)
|
||||
|
||||
catch e
|
||||
close(ts_stream) # Ensure stream is closed on error
|
||||
|
||||
222
test/run_descriptive_preprocessing.jl
Normal file
222
test/run_descriptive_preprocessing.jl
Normal file
@ -0,0 +1,222 @@
|
||||
using CairoMakie
|
||||
using Statistics
|
||||
using Colors
|
||||
|
||||
function create_msi_parameter_plot()
|
||||
# Create the figure with subplots for each preprocessing step
|
||||
fig = Figure(size=(1600, 1200), fontsize=12)
|
||||
|
||||
# Define the preprocessing steps and their parameters
|
||||
steps = [
|
||||
("BaselineCorrection", ["iterations: 10", "method: SNIP", "window: 3.0"]),
|
||||
("Calibration", ["fit_order: 2", "method: internal_standards", "ppm_tolerance: 20.0"]),
|
||||
("Normalization", ["method: rms"]),
|
||||
("PeakAlignment", ["max_shift_ppm: 50.0", "method: linear", "tolerance: 20.0 ppm"]),
|
||||
("PeakBinningParams", ["max_bin_width_ppm: 60.0", "method: adaptive", "min_peak_per_bin: 3"]),
|
||||
("PeakPicking", ["half_window: 5", "method: centroid", "snr_threshold: 15.0"]),
|
||||
("PeakSelection", ["max_fwhm_ppm: 95.39", "min_fwhm_ppm: 19.08", "min_snr: 3.0"]),
|
||||
("Smoothing", ["method: savitzky_golay", "order: 3", "window: 5"])
|
||||
]
|
||||
|
||||
# Create a grid of subplots
|
||||
g = fig[1, 1] = GridLayout()
|
||||
|
||||
# Plot each preprocessing step
|
||||
for (idx, (step_name, params)) in enumerate(steps)
|
||||
row, col = fldmod1(idx, 2)
|
||||
ax = Axis(g[row, col], title=step_name, titlesize=14)
|
||||
|
||||
# Generate simulated m/z values and intensities
|
||||
mz_range = range(100, 1000, length=500)
|
||||
|
||||
if step_name == "BaselineCorrection"
|
||||
# Simulate spectrum with baseline
|
||||
true_signal = 0.5 .* exp.(-0.001 .* (mz_range .- 400).^2) .+
|
||||
0.3 .* exp.(-0.002 .* (mz_range .- 600).^2)
|
||||
baseline = 0.1 .+ 0.05 .* sin.(mz_range ./ 50)
|
||||
noisy_signal = true_signal .+ baseline .+ 0.02 .* randn(length(mz_range))
|
||||
corrected = noisy_signal .- baseline
|
||||
|
||||
lines!(ax, mz_range, noisy_signal, color=:blue, linewidth=2, label="Raw")
|
||||
lines!(ax, mz_range, baseline, color=:red, linewidth=2, linestyle=:dash, label="Baseline")
|
||||
lines!(ax, mz_range, corrected, color=:green, linewidth=2, label="Corrected")
|
||||
|
||||
elseif step_name == "Calibration"
|
||||
# Simulate calibration shift
|
||||
reference_peaks = [200, 400, 600, 800]
|
||||
measured_peaks = reference_peaks .+ 2.0 .* randn(length(reference_peaks))
|
||||
|
||||
scatter!(ax, reference_peaks, fill(0.5, length(reference_peaks)),
|
||||
color=:red, markersize=15, label="Reference")
|
||||
scatter!(ax, measured_peaks, fill(0.3, length(measured_peaks)),
|
||||
color=:blue, markersize=10, label="Measured")
|
||||
|
||||
# Add calibration lines
|
||||
for i in 1:length(reference_peaks)
|
||||
lines!(ax, [measured_peaks[i], reference_peaks[i]], [0.3, 0.5],
|
||||
color=:black, linewidth=1, linestyle=:dash)
|
||||
end
|
||||
|
||||
elseif step_name == "Normalization"
|
||||
# Simulate normalization effect
|
||||
spectra = [
|
||||
0.8 .* exp.(-0.001 .* (mz_range .- 300).^2) .+ 0.2 .* randn(length(mz_range)),
|
||||
1.2 .* exp.(-0.001 .* (mz_range .- 300).^2) .+ 0.2 .* randn(length(mz_range)),
|
||||
0.9 .* exp.(-0.001 .* (mz_range .- 300).^2) .+ 0.2 .* randn(length(mz_range))
|
||||
]
|
||||
|
||||
normalized_spectra = [spec ./ std(spec) for spec in spectra]
|
||||
|
||||
for (i, spec) in enumerate(spectra)
|
||||
lines!(ax, mz_range, spec .+ i*0.3, color=RGBA(1, 0, 0, 0.6), linewidth=2,
|
||||
label=i==1 ? "Before Norm" : "")
|
||||
end
|
||||
for (i, spec) in enumerate(normalized_spectra)
|
||||
lines!(ax, mz_range, spec .+ i*0.3, color=RGBA(0, 0, 1, 0.6), linewidth=2,
|
||||
label=i==1 ? "After Norm" : "")
|
||||
end
|
||||
|
||||
elseif step_name == "PeakAlignment"
|
||||
# Simulate peak alignment
|
||||
base_peaks = [300, 500, 700]
|
||||
shifts = [-15, 5, 10]
|
||||
|
||||
for (i, shift) in enumerate(shifts)
|
||||
shifted_peaks = base_peaks .+ shift
|
||||
aligned_peaks = base_peaks
|
||||
|
||||
scatter!(ax, shifted_peaks, fill(i, length(shifted_peaks)),
|
||||
color=:red, markersize=12, label=i==1 ? "Before Align" : "")
|
||||
scatter!(ax, aligned_peaks, fill(i+0.3, length(aligned_peaks)),
|
||||
color=:green, markersize=12, label=i==1 ? "After Align" : "")
|
||||
|
||||
# Show alignment lines
|
||||
for j in 1:length(base_peaks)
|
||||
lines!(ax, [shifted_peaks[j], aligned_peaks[j]], [i, i+0.3],
|
||||
color=:black, linewidth=1, linestyle=:dash)
|
||||
end
|
||||
end
|
||||
|
||||
elseif step_name == "PeakBinningParams"
|
||||
# Simulate peak binning with centroids
|
||||
raw_peaks_mz = 100:25:900
|
||||
raw_peaks_intensity = rand(length(raw_peaks_mz))
|
||||
|
||||
# Create binned peaks (wider bins)
|
||||
bin_centers = 150:60:850
|
||||
bin_intensities = [sum(raw_peaks_intensity[abs.(raw_peaks_mz .- center) .< 30])
|
||||
for center in bin_centers] .* 0.8
|
||||
|
||||
# Profile mode (continuous)
|
||||
profile_signal = zeros(length(mz_range))
|
||||
for (mz, int) in zip(raw_peaks_mz, raw_peaks_intensity)
|
||||
profile_signal .+= int .* exp.(-0.001 .* (mz_range .- mz).^2)
|
||||
end
|
||||
|
||||
lines!(ax, mz_range, profile_signal, color=:blue, linewidth=2, label="Profile")
|
||||
barplot!(ax, bin_centers, bin_intensities, color=RGBA(1, 0, 0, 0.7),
|
||||
width=50, label="Binned Centroids")
|
||||
|
||||
elseif step_name == "PeakPicking"
|
||||
# Simulate peak picking from profile to centroids
|
||||
profile_signal = 0.6 .* exp.(-0.0005 .* (mz_range .- 400).^2) .+
|
||||
0.4 .* exp.(-0.0008 .* (mz_range .- 650).^2) .+
|
||||
0.1 .* randn(length(mz_range))
|
||||
|
||||
# Simulate picked peaks (centroids)
|
||||
peak_positions = [380, 405, 640, 660]
|
||||
peak_intensities = [0.5, 0.6, 0.35, 0.4]
|
||||
|
||||
lines!(ax, mz_range, profile_signal, color=:blue, linewidth=2, label="Profile Spectrum")
|
||||
scatter!(ax, peak_positions, peak_intensities, color=:red, markersize=20,
|
||||
label="Picked Centroids", strokewidth=2)
|
||||
|
||||
elseif step_name == "PeakSelection"
|
||||
# Simulate peak selection based on criteria
|
||||
all_peaks_mz = 200:50:800
|
||||
all_peaks_fwhm = rand(length(all_peaks_mz)) .* 100 .+ 10
|
||||
all_peaks_snr = rand(length(all_peaks_mz)) .* 10
|
||||
|
||||
# Selection criteria
|
||||
selected = (all_peaks_fwhm .>= 19.08) .& (all_peaks_fwhm .<= 95.39) .& (all_peaks_snr .>= 3.0)
|
||||
|
||||
scatter!(ax, all_peaks_mz[.!selected], all_peaks_fwhm[.!selected],
|
||||
color=:red, markersize=15, label="Rejected")
|
||||
scatter!(ax, all_peaks_mz[selected], all_peaks_fwhm[selected],
|
||||
color=:green, markersize=15, label="Selected")
|
||||
|
||||
# Add selection criteria lines
|
||||
hlines!(ax, [19.08, 95.39], color=:black, linestyle=:dash, linewidth=2)
|
||||
text!(ax, 850, 50; text="FWHM bounds", color=:black, fontsize=10)
|
||||
|
||||
elseif step_name == "Smoothing"
|
||||
# Simulate smoothing effect
|
||||
true_signal = 0.7 .* exp.(-0.001 .* (mz_range .- 450).^2) .+
|
||||
0.5 .* exp.(-0.0008 .* (mz_range .- 650).^2)
|
||||
noisy_signal = true_signal .+ 0.1 .* randn(length(mz_range))
|
||||
|
||||
# Simple smoothing simulation
|
||||
smoothed = similar(noisy_signal)
|
||||
window = 5
|
||||
for i in 1:length(noisy_signal)
|
||||
start_idx = max(1, i - window ÷ 2)
|
||||
end_idx = min(length(noisy_signal), i + window ÷ 2)
|
||||
smoothed[i] = mean(noisy_signal[start_idx:end_idx])
|
||||
end
|
||||
|
||||
lines!(ax, mz_range, noisy_signal, color=:red, linewidth=1, label="Noisy")
|
||||
lines!(ax, mz_range, smoothed, color=:blue, linewidth=2, label="Smoothed")
|
||||
lines!(ax, mz_range, true_signal, color=:green, linewidth=1, linestyle=:dash, label="True")
|
||||
end
|
||||
|
||||
# Add parameter text using a more reliable approach
|
||||
param_text = join(params, "\n")
|
||||
|
||||
text!(ax, param_text, position=Point2f(0.05, 0.95),
|
||||
space=:relative, align=(:left, :top), color=:black,
|
||||
fontsize=10, font=:regular)
|
||||
|
||||
# Add legend for selected plots
|
||||
if idx <= 4
|
||||
axislegend(ax, position=:rt, framevisible=true, backgroundcolor=RGBA(1,1,1,0.8))
|
||||
end
|
||||
|
||||
# Customize axes
|
||||
ax.xlabel = "m/z"
|
||||
ax.ylabel = idx in [1,3,5,7] ? "Intensity" : ""
|
||||
ax.xgridvisible = false
|
||||
ax.ygridvisible = false
|
||||
end
|
||||
|
||||
# Add overall title
|
||||
Label(fig[0, :], "MSI Preprocessing Pipeline Parameters and Simulations",
|
||||
fontsize=18, font=:bold, padding=(0, 0, 10, 0))
|
||||
|
||||
# Add explanation
|
||||
explanation = """
|
||||
Simulation of MSI preprocessing parameters showing:
|
||||
• Blue lines: Profile/continuous spectra
|
||||
• Red bars/points: Centroid data
|
||||
• Dashed lines: Reference/true signals
|
||||
• Green: Processed/corrected data
|
||||
• Each subplot demonstrates key parameters for the preprocessing step
|
||||
"""
|
||||
|
||||
Label(fig[2, :], explanation, fontsize=12, tellwidth=false, padding=(10, 10, 10, 10))
|
||||
|
||||
# Adjust layout
|
||||
colgap!(g, 20)
|
||||
rowgap!(g, 20)
|
||||
|
||||
fig
|
||||
end
|
||||
|
||||
# Create and display the plot
|
||||
fig = create_msi_parameter_plot()
|
||||
|
||||
# Save the plot
|
||||
save("msi_preprocessing_parameters.png", fig)
|
||||
println("Plot saved as 'msi_preprocessing_parameters.png'")
|
||||
|
||||
# Display the plot (if in an interactive environment)
|
||||
fig
|
||||
157
test/run_precalculation_dict.jl
Normal file
157
test/run_precalculation_dict.jl
Normal file
@ -0,0 +1,157 @@
|
||||
# test/run_precalculation_example.jl
|
||||
|
||||
using Printf
|
||||
import Pkg
|
||||
|
||||
# --- Load the MSI_src Module ---
|
||||
Pkg.activate(joinpath(@__DIR__, ".."))
|
||||
using MSI_src
|
||||
|
||||
# ===================================================================
|
||||
# CONFIG: PLEASE FILL IN YOUR FILE PATHS HERE
|
||||
# ===================================================================
|
||||
|
||||
# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML"
|
||||
#const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/mzML/Col_1.mzML"
|
||||
const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.mzML"
|
||||
|
||||
# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.imzML"
|
||||
#const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML"
|
||||
const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
|
||||
|
||||
#const MASK_ROUTE = "/home/pixel/Documents/Cinvestav_2025/JuliaMSI/public/css/masks/Stomach_DHB_uncompressed.png"
|
||||
const MASK_ROUTE = ""
|
||||
|
||||
const reference_peaks = Dict(
|
||||
# DHB Matrix peaks (should be present)
|
||||
137.0244 => "DHB_fragment",
|
||||
155.0349 => "DHB_M+H",
|
||||
177.0168 => "DHB_M+Na",
|
||||
|
||||
# Common lipids in your mass range
|
||||
496.3398 => "PC_16:0_16:0",
|
||||
520.3398 => "PC_16:0_18:1",
|
||||
760.5851 => "PC_16:0_18:1_Na",
|
||||
|
||||
# Common contaminants
|
||||
391.2843 => "PDMS",
|
||||
413.2662 => "PDMS_Na",
|
||||
|
||||
# Add some high mass peaks
|
||||
842.5092 => "Protein_standard",
|
||||
1045.532 => "Protein_standard",
|
||||
|
||||
290.1747 => "Atropine [M+H]+",
|
||||
304.1903 => "Scopolamine [M+H]+",
|
||||
124.0393 => "Tropine [M+H]+",
|
||||
)
|
||||
|
||||
# ===================================================================
|
||||
# HELPER FUNCTIONS FOR PRINTING
|
||||
# ===================================================================
|
||||
|
||||
function print_header(title::String)
|
||||
println("\n" * "="^80)
|
||||
println("$(title)")
|
||||
println("="^80)
|
||||
end
|
||||
|
||||
function check_data_range(msi_data::MSIData)
|
||||
println("\n--- Data Range Analysis ---")
|
||||
|
||||
min_mz, max_mz = get_global_mz_range(msi_data)
|
||||
if isfinite(min_mz) && isfinite(max_mz) && min_mz < max_mz
|
||||
println("Global m/z range: [$(min_mz), $(max_mz)]")
|
||||
else
|
||||
println("Global m/z range: Not yet determined or invalid (initial: [$(min_mz), $(max_mz)])")
|
||||
end
|
||||
|
||||
# Check a few spectra to see actual m/z values
|
||||
println("\nChecking first few spectra for actual m/z values:")
|
||||
for i in 1:min(3, length(msi_data.spectra_metadata))
|
||||
try
|
||||
mz, intensity = GetSpectrum(msi_data, i)
|
||||
if !isempty(mz)
|
||||
println("Spectrum $i: m/z range [$(minimum(mz)), $(maximum(mz))], length=$(length(mz))")
|
||||
# Print first and last few m/z values
|
||||
if length(mz) > 10
|
||||
println(" First 5 m/z: $(mz[1:5])")
|
||||
println(" Last 5 m/z: $(mz[end-4:end])")
|
||||
end
|
||||
end
|
||||
catch e
|
||||
println("Spectrum $i: Error - $e")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# ===================================================================
|
||||
# MAIN EXAMPLE RUNNER
|
||||
# ===================================================================
|
||||
|
||||
function run_precalculation_example()
|
||||
# --- Process mzML file ---
|
||||
print_header("Processing mzML File: $(basename(TEST_MZML_FILE))")
|
||||
if !isfile(TEST_MZML_FILE)
|
||||
println("SKIPPING: mzML file not found at $(TEST_MZML_FILE)")
|
||||
else
|
||||
try
|
||||
msi_data_mzml = @time OpenMSIData(TEST_MZML_FILE)
|
||||
check_data_range(msi_data_mzml)
|
||||
analysis_results_mzml = main_precalculation(msi_data_mzml, reference_peaks=reference_peaks)
|
||||
|
||||
println("\n" * "*"^80)
|
||||
println("MZML PREPROCESSING ANALYSIS RESULTS")
|
||||
println("*"^80)
|
||||
for (step_name, params) in analysis_results_mzml
|
||||
println("\n--- $(uppercase(string(step_name))) Parameters ---")
|
||||
if isempty(params)
|
||||
println(" No recommended parameters.")
|
||||
else
|
||||
for (param_name, param_value) in params
|
||||
println(" $param_name: $param_value")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
close(msi_data_mzml) # Close file handles
|
||||
catch e
|
||||
println("ERROR processing mzML file: $e")
|
||||
showerror(stdout, e, catch_backtrace())
|
||||
end
|
||||
end
|
||||
|
||||
# --- Process imzML file ---
|
||||
print_header("Processing imzML File: $(basename(TEST_IMZML_FILE))")
|
||||
if !isfile(TEST_IMZML_FILE)
|
||||
println("SKIPPING: imzML file not found at $(TEST_IMZML_FILE)")
|
||||
else
|
||||
try
|
||||
msi_data_imzml = @time OpenMSIData(TEST_IMZML_FILE)
|
||||
check_data_range(msi_data_imzml)
|
||||
analysis_results_imzml = main_precalculation(msi_data_imzml, reference_peaks=reference_peaks, mask_path=MASK_ROUTE)
|
||||
|
||||
println("\n" * "*"^80)
|
||||
println("IMZML PREPROCESSING ANALYSIS RESULTS")
|
||||
println("*"^80)
|
||||
for (step_name, params) in analysis_results_imzml
|
||||
println("\n--- $(uppercase(string(step_name))) Parameters ---")
|
||||
if isempty(params)
|
||||
println(" No recommended parameters.")
|
||||
else
|
||||
for (param_name, param_value) in params
|
||||
println(" $param_name: $param_value")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
close(msi_data_imzml) # Close file handles
|
||||
catch e
|
||||
println("ERROR processing imzML file: $e")
|
||||
showerror(stdout, e, catch_backtrace())
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# --- Execute ---
|
||||
@time run_precalculation_example()
|
||||
281
test/run_precalculation_example.jl
Normal file
281
test/run_precalculation_example.jl
Normal file
@ -0,0 +1,281 @@
|
||||
# test/run_precalculation_example.jl
|
||||
|
||||
using Printf
|
||||
import Pkg
|
||||
|
||||
# --- Load the MSI_src Module ---
|
||||
Pkg.activate(joinpath(@__DIR__, ".."))
|
||||
using MSI_src
|
||||
|
||||
# ===================================================================
|
||||
# CONFIG: PLEASE FILL IN YOUR FILE PATHS HERE
|
||||
# ===================================================================
|
||||
|
||||
# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML"
|
||||
# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/mzML/Col_1.mzML"
|
||||
const TEST_MZML_FILE = ""
|
||||
# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.imzML"
|
||||
#const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML"
|
||||
const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
|
||||
#const MASK_ROUTE = "/home/pixel/Documents/Cinvestav_2025/JuliaMSI/public/css/masks/Stomach_DHB_uncompressed.png"
|
||||
const MASK_ROUTE = ""
|
||||
|
||||
#=
|
||||
const reference_peaks = Dict(
|
||||
# Common ESI positive mode reference compounds
|
||||
121.0509 => "Purine",
|
||||
149.0233 => "HP-921",
|
||||
322.0481 => "Hexakis(1H,1H,3H-tetrafluoropropoxy)phosphazine",
|
||||
622.0290 => "Hexakis(2,2-difluoroethoxy)phosphazine",
|
||||
|
||||
# Atropine and related compounds in positive mode
|
||||
290.1747 => "Atropine [M+H]+",
|
||||
304.1903 => "Scopolamine [M+H]+",
|
||||
124.0393 => "Tropine [M+H]+",
|
||||
|
||||
# Common contaminants and lock masses
|
||||
391.2843 => "Polydimethylcyclosiloxane [M+H]+",
|
||||
413.2662 => "Polydimethylcyclosiloxane [M+Na]+",
|
||||
429.2402 => "Polydimethylcyclosiloxane [M+K]+"
|
||||
)
|
||||
=#
|
||||
|
||||
const reference_peaks = Dict(
|
||||
# DHB Matrix peaks (should be present)
|
||||
137.0244 => "DHB_fragment",
|
||||
155.0349 => "DHB_M+H",
|
||||
177.0168 => "DHB_M+Na",
|
||||
|
||||
# Common lipids in your mass range
|
||||
496.3398 => "PC_16:0_16:0",
|
||||
520.3398 => "PC_16:0_18:1",
|
||||
760.5851 => "PC_16:0_18:1_Na",
|
||||
|
||||
# Common contaminants
|
||||
391.2843 => "PDMS",
|
||||
413.2662 => "PDMS_Na",
|
||||
|
||||
# Add some high mass peaks
|
||||
842.5092 => "Protein_standard",
|
||||
1045.532 => "Protein_standard",
|
||||
)
|
||||
|
||||
# ===================================================================
|
||||
# HELPER FUNCTIONS FOR PRINTING
|
||||
# ===================================================================
|
||||
|
||||
function print_header(title::String)
|
||||
println("\n" * "="^80)
|
||||
println("$(title)")
|
||||
println("="^80)
|
||||
end
|
||||
|
||||
function print_subheader(title::String)
|
||||
println("\n" * "-"^80)
|
||||
println("$(title)")
|
||||
println("-"^80)
|
||||
end
|
||||
|
||||
function print_param(key, value)
|
||||
if value === nothing || (isa(value, Number) && isnan(value))
|
||||
println(" " * "" * rpad(key, 30) * ": unable to get, user needs to input manually")
|
||||
elseif isa(value, Symbol) && startswith(string(key), "method") # Heuristic for method selection
|
||||
println(" " * "" * rpad(key, 30) * ": automatically detected this as the most optimal method: $(value)")
|
||||
else
|
||||
println(" " * "" * rpad(key, 30) * ": $(value)")
|
||||
end
|
||||
end
|
||||
|
||||
function print_recommendations(recommendations::Dict)
|
||||
for (step, params) in recommendations
|
||||
print_subheader("Recommendations for $(String(step))")
|
||||
for (key, value) in params
|
||||
print_param(key, value)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function check_data_range(msi_data::MSIData)
|
||||
println("\n--- Data Range Analysis ---")
|
||||
|
||||
min_mz, max_mz = get_global_mz_range(msi_data)
|
||||
if isfinite(min_mz) && isfinite(max_mz) && min_mz < max_mz
|
||||
println("Global m/z range: [$(min_mz), $(max_mz)]")
|
||||
else
|
||||
println("Global m/z range: Not yet determined or invalid (initial: [$(min_mz), $(max_mz)])")
|
||||
end
|
||||
|
||||
# Check a few spectra to see actual m/z values
|
||||
println("\nChecking first few spectra for actual m/z values:")
|
||||
for i in 1:min(3, length(msi_data.spectra_metadata))
|
||||
try
|
||||
mz, intensity = GetSpectrum(msi_data, i)
|
||||
if !isempty(mz)
|
||||
println("Spectrum $i: m/z range [$(minimum(mz)), $(maximum(mz))], length=$(length(mz))")
|
||||
# Print first and last few m/z values
|
||||
if length(mz) > 10
|
||||
println(" First 5 m/z: $(mz[1:5])")
|
||||
println(" Last 5 m/z: $(mz[end-4:end])")
|
||||
end
|
||||
end
|
||||
catch e
|
||||
println("Spectrum $i: Error - $e")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# ===================================================================
|
||||
# MAIN EXAMPLE RUNNER
|
||||
# ===================================================================
|
||||
|
||||
function run_precalculation_example()
|
||||
# --- Process mzML file ---
|
||||
print_header("Processing mzML File: $(basename(TEST_MZML_FILE))")
|
||||
if !isfile(TEST_MZML_FILE)
|
||||
println("SKIPPING: mzML file not found at $(TEST_MZML_FILE)")
|
||||
else
|
||||
try
|
||||
msi_data_mzml = @time OpenMSIData(TEST_MZML_FILE)
|
||||
check_data_range(msi_data_mzml)
|
||||
analysis_results_mzml = run_preprocessing_analysis(msi_data_mzml, reference_peaks=reference_peaks)
|
||||
|
||||
println("\n" * "*"^80)
|
||||
println("MZML PREPROCESSING ANALYSIS RESULTS")
|
||||
println("*"^80)
|
||||
|
||||
for (phase, results) in analysis_results_mzml
|
||||
if phase == :recommendations
|
||||
print_subheader("Generated Preprocessing Recommendations")
|
||||
print_recommendations(results)
|
||||
else
|
||||
print_subheader("Phase: $(String(phase))")
|
||||
if isa(results, Dict)
|
||||
for (key, value) in results
|
||||
print_param(key, value)
|
||||
end
|
||||
elseif isa(results, NamedTuple)
|
||||
for field in fieldnames(typeof(results))
|
||||
print_param(field, getfield(results, field))
|
||||
end
|
||||
else
|
||||
println(" $(String(phase)) results: $(results)")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
close(msi_data_mzml) # Close file handles
|
||||
catch e
|
||||
println("ERROR processing mzML file: $e")
|
||||
showerror(stdout, e, catch_backtrace())
|
||||
end
|
||||
end
|
||||
|
||||
# --- Process imzML file ---
|
||||
print_header("Processing imzML File: $(basename(TEST_IMZML_FILE))")
|
||||
if !isfile(TEST_IMZML_FILE)
|
||||
println("SKIPPING: imzML file not found at $(TEST_IMZML_FILE)")
|
||||
else
|
||||
try
|
||||
msi_data_imzml = @time OpenMSIData(TEST_IMZML_FILE)
|
||||
check_data_range(msi_data_imzml)
|
||||
analysis_results_imzml = run_preprocessing_analysis(msi_data_imzml, reference_peaks=reference_peaks, mask_path=MASK_ROUTE)
|
||||
|
||||
println("\n" * "*"^80)
|
||||
println("IMZML PREPROCESSING ANALYSIS RESULTS")
|
||||
println("*"^80)
|
||||
|
||||
for (phase, results) in analysis_results_imzml
|
||||
if phase == :recommendations
|
||||
print_subheader("Generated Preprocessing Recommendations")
|
||||
print_recommendations(results)
|
||||
else
|
||||
print_subheader("Phase: $(String(phase))")
|
||||
if isa(results, Dict)
|
||||
for (key, value) in results
|
||||
print_param(key, value)
|
||||
end
|
||||
elseif isa(results, NamedTuple)
|
||||
for field in fieldnames(typeof(results))
|
||||
print_param(field, getfield(results, field))
|
||||
end
|
||||
else
|
||||
println(" $(String(phase)) results: $(results)")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
close(msi_data_imzml) # Close file handles
|
||||
catch e
|
||||
println("ERROR processing imzML file: $e")
|
||||
showerror(stdout, e, catch_backtrace())
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# --- Execute ---
|
||||
@time run_precalculation_example()
|
||||
|
||||
# =============================================================================
|
||||
# Example Usage
|
||||
# =============================================================================
|
||||
|
||||
#=
|
||||
"""
|
||||
example_preanalysis_workflow(msi_data::MSIData)
|
||||
|
||||
Demonstrates how to run the pre-analysis pipeline.
|
||||
"""
|
||||
function example_preanalysis_workflow(msi_data::MSIData)
|
||||
# Load your MSIData object (this would come from your actual data loading)
|
||||
# msi_data = load_imzml_dataset("path/to/your/data.imzML") # Assuming msi_data is passed
|
||||
|
||||
# Define reference peaks for mass accuracy analysis
|
||||
reference_peaks = Dict(
|
||||
89.04767 => "Alanin",
|
||||
147.07642 => "Lysin",
|
||||
189.12392 => "Unknown",
|
||||
524.26496 => "PC(34:1) [M+H]+"
|
||||
)
|
||||
|
||||
# Define region masks if you have spatial annotations
|
||||
region_masks = Dict{Symbol, BitMatrix}()
|
||||
# region_masks[:tumor] = load_mask("tumor_mask.png") # Not defined, comment out
|
||||
# region_masks[:stroma] = load_mask("stroma_mask.png") # Not defined, comment out
|
||||
|
||||
println("Starting comprehensive pre-analysis...")
|
||||
|
||||
# Run the complete pre-analysis pipeline
|
||||
analysis_results = run_preprocessing_analysis(
|
||||
msi_data, # Your MSIData object
|
||||
reference_peaks=reference_peaks,
|
||||
region_masks=region_masks,
|
||||
sample_size=200 # Adjust based on dataset size
|
||||
)
|
||||
|
||||
# Access the recommendations
|
||||
recommendations = analysis_results[:recommendations]
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("PREPROCESSING RECOMMENDATIONS")
|
||||
println("="^60)
|
||||
|
||||
for (step, params) in recommendations
|
||||
println("\n$step:")
|
||||
for (key, value) in params
|
||||
println(" - $key: $value")
|
||||
end
|
||||
end
|
||||
|
||||
return analysis_results
|
||||
end
|
||||
|
||||
# You can also run individual analysis steps:
|
||||
function run_targeted_analysis(msi_data::MSIData)
|
||||
# Just analyze signal quality and peak characteristics
|
||||
signal_analysis = analyze_signal_quality(msi_data, sample_size=100)
|
||||
peak_analysis = analyze_peak_characteristics(msi_data, sample_size=50)
|
||||
|
||||
return (signal_analysis, peak_analysis)
|
||||
end
|
||||
|
||||
=#
|
||||
@ -1,371 +1,534 @@
|
||||
# test/run_preprocessing.jl
|
||||
|
||||
# ===================================================================
|
||||
# Preprocessing Test Suite for JuliaMSI
|
||||
# ===================================================================
|
||||
# This script provides a customizable workflow to test and visualize
|
||||
# the effects of different preprocessing steps on individual spectra
|
||||
# from .mzML or .imzML files.
|
||||
#
|
||||
# Instructions:
|
||||
# 1. Configure the file paths and parameters in the "CONFIG" section.
|
||||
# 2. Run the script from the project's root directory:
|
||||
# julia --threads auto --project=. test/run_preprocessing.jl
|
||||
# 3. Check the configured `RESULTS_DIR` for output plots and CSV files.
|
||||
# ===================================================================
|
||||
# run_preprocessing.jl
|
||||
|
||||
using Printf
|
||||
using CairoMakie
|
||||
using DataFrames
|
||||
using CSV
|
||||
using Statistics
|
||||
using Interpolations
|
||||
import Pkg
|
||||
using CairoMakie
|
||||
using DataFrames # For creating dataframes
|
||||
using CSV
|
||||
|
||||
# --- Load the MSI_src Module ---
|
||||
# Activate the project environment at the parent directory of this test script
|
||||
Pkg.activate(joinpath(@__DIR__, ".."))
|
||||
using MSI_src
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# CONFIG: CUSTOMIZE YOUR PREPROCESSING WORKFLOW HERE
|
||||
# CONFIG: PLEASE FILL IN YOUR FILE PATHS HERE
|
||||
# ===================================================================
|
||||
|
||||
# --- Input and Output ---
|
||||
const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML" # Can be .mzML or .imzML
|
||||
const RESULTS_DIR = "test/results/preprocessing"
|
||||
const NUM_SPECTRA_TO_PROCESS = nothing # Set to `nothing` to process all spectra
|
||||
const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
|
||||
# const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/set de datos MS/Atropina_tuneo_fraq_20ev.mzML"
|
||||
# const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML"
|
||||
|
||||
# --- Internal Standards for Calibration and QC ---
|
||||
# replace these with m/z values of known compounds present in your dataset.
|
||||
const INTERNAL_STANDARDS = Dict(
|
||||
"P13" => 432.6584,
|
||||
"P15" => 464.6059,
|
||||
"P17" => 526.5534,
|
||||
"P21" => 650.4485,
|
||||
"P25" => 774.3435,
|
||||
"P29" => 898.2385,
|
||||
"P31" => 950.1861,
|
||||
"P33" => 1022.1336,
|
||||
"P37" => 1146.0286,
|
||||
"P45" => 1593.8187,
|
||||
"Unknown 1" => 772.433,
|
||||
"Unknown 2" => 772.5253
|
||||
)
|
||||
# const MASK_ROUTE = "/home/pixel/Documents/Cinvestav_2025/JuliaMSI/public/css/masks/Stomach_DHB_uncompressed.png"
|
||||
const MASK_ROUTE = ""
|
||||
|
||||
# --- Preprocessing Step Parameters ---
|
||||
const OUTPUT_DIR = "./test/results/preprocessing_results"
|
||||
|
||||
# Step 0: Quality Control (QC)
|
||||
const QC_PARAMS = (
|
||||
ppm_tolerance = 5.0, # PPM tolerance for matching internal standards
|
||||
)
|
||||
|
||||
# Step 1: Calibration
|
||||
const CALIBRATION_PARAMS = (
|
||||
enabled = true,
|
||||
ppm_tolerance = 5.0, # PPM tolerance for finding calibration peaks
|
||||
)
|
||||
|
||||
# Step 2: Smoothing
|
||||
# Note: Smoothing is less effective and often unnecessary for centroid data.
|
||||
const SMOOTHING_PARAMS = (
|
||||
enabled = false,
|
||||
window = 9,
|
||||
order = 2,
|
||||
)
|
||||
|
||||
# Step 3: Baseline Correction
|
||||
# Note: Baseline correction is less effective and often unnecessary for centroid data.
|
||||
const BASELINE_CORRECTION_PARAMS = (
|
||||
enabled = false,
|
||||
iterations = 100,
|
||||
)
|
||||
|
||||
# Step 4: Normalization
|
||||
const NORMALIZATION_PARAMS = (
|
||||
enabled = true,
|
||||
method = :tic, # :tic, :median, or :none
|
||||
)
|
||||
|
||||
# Step 5: Peak Detection
|
||||
const PEAK_DETECTION_PARAMS = (
|
||||
enabled = true,
|
||||
method = :centroid, # Use :centroid for centroided data, :profile for profile data
|
||||
# --- Parameters for :centroid method ---
|
||||
intensity_threshold = 0.0, # Filters out peaks below this absolute intensity
|
||||
# --- Parameters for :profile method ---
|
||||
half_window = 10,
|
||||
snr_threshold = 3.0,
|
||||
min_peak_prominence = 0.05,
|
||||
merge_peaks_tolerance = 0.002,
|
||||
)
|
||||
|
||||
# Step 6: Peak Alignment (Warping)
|
||||
# This is performed after collecting peaks from all spectra.
|
||||
const PEAK_ALIGNMENT_PARAMS = (
|
||||
enabled = true,
|
||||
tolerance = 0.002,
|
||||
tolerance_unit = :mz, # :mz or :ppm
|
||||
min_matched_peaks = 3, # Lowered for sparse centroid data
|
||||
)
|
||||
|
||||
# Step 7: Peak Binning
|
||||
const PEAK_BINNING_PARAMS = (
|
||||
enabled = true,
|
||||
tolerance = 0.1,
|
||||
tolerance_unit = :mz, # :mz or :ppm
|
||||
frequency_threshold = 0.1, # Min fraction of spectra a peak must be in
|
||||
# Reference peaks for calibration and alignment
|
||||
const reference_peaks = Dict(
|
||||
137.0244 => "DHB_fragment",
|
||||
155.0349 => "DHB_M+H",
|
||||
177.0168 => "DHB_M+Na",
|
||||
496.3398 => "PC_16:0_16:0",
|
||||
520.3398 => "PC_16:0_18:1",
|
||||
760.5851 => "PC_16:0_18:1_Na",
|
||||
391.2843 => "PDMS",
|
||||
413.2662 => "PDMS_Na",
|
||||
842.5092 => "Protein_standard",
|
||||
1045.532 => "Protein_standard",
|
||||
290.1747 => "Atropine [M+H]+",
|
||||
304.1903 => "Scopolamine [M+H]+",
|
||||
124.0393 => "Tropine [M+H]+",
|
||||
)
|
||||
|
||||
# ===================================================================
|
||||
# HELPER FUNCTIONS
|
||||
# ===================================================================
|
||||
|
||||
"""
|
||||
plot_spectrum_on_fig(fig_pos, mz, intensity, title)
|
||||
|
||||
Helper function to plot a spectrum on a specific position of a CairoMakie figure.
|
||||
"""
|
||||
function plot_spectrum_on_fig(fig_pos, mz, intensity, title)
|
||||
ax = Axis(fig_pos, title=title, xlabel="m/z", ylabel="Intensity")
|
||||
lines!(ax, mz, intensity)
|
||||
function ensure_output_dir()
|
||||
if !isdir(OUTPUT_DIR)
|
||||
mkdir(OUTPUT_DIR)
|
||||
println("Created output directory: $OUTPUT_DIR")
|
||||
end
|
||||
end
|
||||
|
||||
function plot_spectrum_step(mz, intensity, step_name; peaks=nothing, spectrum_index=1)
|
||||
fig = Figure(size=(1200, 600))
|
||||
ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Intensity", title="Step: $step_name (Spectrum $spectrum_index)")
|
||||
|
||||
# ===================================================================
|
||||
# MAIN PROCESSING SCRIPT
|
||||
# ===================================================================
|
||||
lines!(ax, mz, intensity, color=:blue, linewidth=1)
|
||||
|
||||
function run_preprocessing_suite()
|
||||
println("Starting Preprocessing Test Suite...")
|
||||
mkpath(RESULTS_DIR)
|
||||
|
||||
# --- Load Data ---
|
||||
println("Loading data from: $TEST_FILE")
|
||||
if !isfile(TEST_FILE)
|
||||
println("ERROR: Test file not found. Please check the path in the CONFIG section.")
|
||||
return
|
||||
if peaks !== nothing && !isempty(peaks)
|
||||
peak_mz = [p.mz for p in peaks]
|
||||
peak_intensity = [p.intensity for p in peaks]
|
||||
scatter!(ax, peak_mz, peak_intensity, color=:red, markersize=5)
|
||||
end
|
||||
msi_data = @time OpenMSIData(TEST_FILE)
|
||||
println("Data loaded successfully. Found $(length(msi_data.spectra_metadata)) spectra.")
|
||||
|
||||
# --- Determine which spectra to process ---
|
||||
total_spectra = length(msi_data.spectra_metadata)
|
||||
indices_to_process = if NUM_SPECTRA_TO_PROCESS === nothing
|
||||
1:total_spectra
|
||||
filename = joinpath(OUTPUT_DIR, "spectrum_$(step_name)_index_$spectrum_index.png")
|
||||
save(filename, fig)
|
||||
println(" - Saved plot: $(basename(filename))")
|
||||
end
|
||||
|
||||
function print_step_header(step_name::String)
|
||||
println("\n" * ">"^50)
|
||||
println("APPLYING STEP: $step_name")
|
||||
println(">"^50)
|
||||
end
|
||||
|
||||
# ===================================================================
|
||||
# PREPROCESSING PIPELINE FUNCTIONS (IN-PLACE)
|
||||
# ===================================================================
|
||||
|
||||
function apply_baseline_correction(spectra::Vector{MutableSpectrum}, params::Dict, msi_data::MSIData)
|
||||
print_step_header("Baseline Correction")
|
||||
|
||||
method = get(params, :method, :snip)
|
||||
iterations = get(params, :iterations, 100)
|
||||
window = get(params, :window, 20)
|
||||
println(" Method: $method, Iterations: $iterations, Window: $window")
|
||||
|
||||
# Safely plot the first spectrum
|
||||
if !isempty(spectra)
|
||||
s = spectra[1]
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
baseline = MSI_src.apply_baseline_correction(s.intensity; method=method, iterations=iterations, window=window)
|
||||
corrected_intensity = max.(0.0, s.intensity .- baseline)
|
||||
plot_spectrum_step(s.mz, corrected_intensity, "baseline_correction")
|
||||
end
|
||||
end
|
||||
|
||||
Threads.@threads for s in spectra
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
baseline = MSI_src.apply_baseline_correction(s.intensity; method=method, iterations=iterations, window=window)
|
||||
s.intensity = max.(0.0, s.intensity .- baseline)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function apply_smoothing(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
print_step_header("Smoothing")
|
||||
|
||||
method = get(params, :method, :savitzky_golay)
|
||||
window = get(params, :window, 9)
|
||||
order = get(params, :order, 2)
|
||||
println(" Method: $method, Window: $window, Order: $order")
|
||||
|
||||
# Safely plot the first spectrum
|
||||
if !isempty(spectra)
|
||||
s = spectra[1]
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
smoothed_intensity = max.(0.0, smooth_spectrum(s.intensity; method=method, window=window, order=order))
|
||||
plot_spectrum_step(s.mz, smoothed_intensity, "smoothing", spectrum_index=s.id)
|
||||
end
|
||||
end
|
||||
|
||||
Threads.@threads for s in spectra
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
smoothed_intensity = max.(0.0, smooth_spectrum(s.intensity; method=method, window=window, order=order))
|
||||
s.intensity = smoothed_intensity
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
print_step_header("Peak Picking")
|
||||
|
||||
method = get(params, :method, :profile)
|
||||
snr_threshold = get(params, :snr_threshold, 3.0)
|
||||
half_window = get(params, :half_window, 10)
|
||||
min_peak_prominence = get(params, :min_peak_prominence, 0.1)
|
||||
merge_peaks_tolerance = get(params, :merge_peaks_tolerance, 0.002)
|
||||
println(" Method: $method, SNR: $snr_threshold, Half Window: $half_window")
|
||||
|
||||
found_invalid_spectrum = Threads.Atomic{Bool}(false)
|
||||
|
||||
Threads.@threads for s in spectra
|
||||
is_valid = validate_spectrum(s.mz, s.intensity)
|
||||
|
||||
if !is_valid && !found_invalid_spectrum[]
|
||||
if Threads.atomic_xchg!(found_invalid_spectrum, true) == false
|
||||
# Debug printing for the first invalid spectrum found
|
||||
end
|
||||
end
|
||||
|
||||
if is_valid
|
||||
if method == :profile
|
||||
s.peaks = detect_peaks_profile(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window, min_peak_prominence=min_peak_prominence, merge_peaks_tolerance=merge_peaks_tolerance)
|
||||
elseif method == :wavelet
|
||||
s.peaks = detect_peaks_wavelet(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
|
||||
elseif method == :centroid
|
||||
s.peaks = detect_peaks_centroid(s.mz, s.intensity; snr_threshold=snr_threshold)
|
||||
else
|
||||
unique(round.(Int, range(1, total_spectra, length=min(NUM_SPECTRA_TO_PROCESS, total_spectra))))
|
||||
s.peaks = detect_peaks_profile(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
|
||||
end
|
||||
else
|
||||
s.peaks = []
|
||||
end
|
||||
end
|
||||
println("Will process $(length(indices_to_process)) spectra.")
|
||||
|
||||
# --- Data storage for results ---
|
||||
qc_results = DataFrame(spectrum_idx=Int[], metric=String[], value=Float64[], compound=String[])
|
||||
all_processed_peaks_mz = Vector{Vector{Float64}}()
|
||||
all_processed_peaks_int = Vector{Vector{Float64}}()
|
||||
spectrum_indices_with_peaks = Int[]
|
||||
# Safely plot the first spectrum
|
||||
if !isempty(spectra)
|
||||
s1 = findfirst(s -> s.id == 1, spectra)
|
||||
if s1 !== nothing
|
||||
plot_spectrum_step(spectra[s1].mz, spectra[s1].intensity, "peak_picking", peaks=spectra[s1].peaks)
|
||||
println(" - Detected $(length(spectra[s1].peaks)) peaks in spectrum 1")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# --- Main Loop: Process each spectrum individually ---
|
||||
println("\n" * "="^20 * " Processing Individual Spectra " * "="^20)
|
||||
for (i, idx) in enumerate(indices_to_process)
|
||||
print("\rProcessing spectrum #$idx ($(i)/$(length(indices_to_process)))...")
|
||||
function apply_calibration(spectra::Vector{MutableSpectrum}, params::Dict, reference_peaks::Dict)
|
||||
print_step_header("Calibration")
|
||||
|
||||
process_spectrum(msi_data, idx) do mz, intensity
|
||||
if qc_is_empty(mz, intensity) || !qc_is_regular(mz)
|
||||
# @warn "Skipping empty or irregular spectrum #$idx"
|
||||
method = get(params, :method, :none)
|
||||
ppm_tolerance = get(params, :ppm_tolerance, 20.0)
|
||||
println(" Method: $method, PPM Tolerance: $ppm_tolerance")
|
||||
|
||||
if method == :none || isempty(reference_peaks)
|
||||
println(" - Skipping calibration (no method or reference peaks)")
|
||||
return
|
||||
end
|
||||
|
||||
original_mz, original_intensity = copy(mz), copy(intensity)
|
||||
processed_mz, processed_intensity = copy(mz), copy(intensity)
|
||||
reference_masses = collect(keys(reference_peaks))
|
||||
calibration_info = Vector{String}(undef, length(spectra))
|
||||
|
||||
# --- Step 0: QC (PPM and Resolution) ---
|
||||
# This happens before any modification to the m/z axis
|
||||
let ref_masses = collect(values(INTERNAL_STANDARDS))
|
||||
matched_peaks = find_calibration_peaks(processed_mz, processed_intensity, ref_masses, ppm_tolerance=QC_PARAMS.ppm_tolerance)
|
||||
|
||||
for (compound, theoretical_mz) in INTERNAL_STANDARDS
|
||||
# Find the measured m/z that corresponds to this theoretical_mz
|
||||
measured_mz = 0.0
|
||||
for (theo, meas) in matched_peaks
|
||||
if theo == theoretical_mz
|
||||
measured_mz = meas
|
||||
break
|
||||
end
|
||||
end
|
||||
|
||||
if measured_mz > 0
|
||||
# PPM Error
|
||||
ppm_error = calculate_ppm_error(measured_mz, theoretical_mz)
|
||||
push!(qc_results, (idx, "ppm_error", ppm_error, compound))
|
||||
|
||||
# Resolution
|
||||
resolution = calculate_resolution_fwhm(measured_mz, processed_mz, processed_intensity)
|
||||
if !isnan(resolution)
|
||||
push!(qc_results, (idx, "resolution", resolution, compound))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# --- Step 1: Calibration ---
|
||||
if CALIBRATION_PARAMS.enabled
|
||||
ref_masses = collect(values(INTERNAL_STANDARDS))
|
||||
matched_peaks = find_calibration_peaks(processed_mz, processed_intensity, ref_masses, ppm_tolerance=CALIBRATION_PARAMS.ppm_tolerance)
|
||||
Threads.@threads for i in 1:length(spectra)
|
||||
s = spectra[i]
|
||||
info_message = ""
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
matched_peaks = find_calibration_peaks(s.mz, s.intensity, reference_masses; ppm_tolerance=ppm_tolerance)
|
||||
if length(matched_peaks) >= 2
|
||||
measured = sort(collect(values(matched_peaks)))
|
||||
theoretical = sort(collect(keys(matched_peaks)))
|
||||
itp = linear_interpolation(measured, theoretical, extrapolation_bc=Line())
|
||||
processed_mz = itp(processed_mz)
|
||||
s.mz = itp(s.mz) # Modify mz-axis in-place
|
||||
info_message = " - Spectrum $(s.id): calibrated using $(length(matched_peaks)) reference peaks"
|
||||
else
|
||||
info_message = " - Spectrum $(s.id): insufficient reference peaks ($(length(matched_peaks)) found), skipping"
|
||||
end
|
||||
end
|
||||
calibration_info[i] = info_message
|
||||
end
|
||||
|
||||
# --- Step 2: Smoothing ---
|
||||
if SMOOTHING_PARAMS.enabled
|
||||
processed_intensity = smooth_spectrum(processed_intensity, window=SMOOTHING_PARAMS.window, order=SMOOTHING_PARAMS.order)
|
||||
end
|
||||
|
||||
# --- Step 3: Baseline Correction ---
|
||||
if BASELINE_CORRECTION_PARAMS.enabled
|
||||
baseline = snip_baseline(processed_intensity, iterations=BASELINE_CORRECTION_PARAMS.iterations)
|
||||
processed_intensity .-= baseline
|
||||
processed_intensity = max.(0, processed_intensity) # Ensure non-negativity
|
||||
end
|
||||
|
||||
# --- Step 4: Normalization ---
|
||||
if NORMALIZATION_PARAMS.enabled && NORMALIZATION_PARAMS.method != :none
|
||||
if NORMALIZATION_PARAMS.method == :tic
|
||||
processed_intensity = tic_normalize(processed_intensity)
|
||||
elseif NORMALIZATION_PARAMS.method == :median
|
||||
processed_intensity = median_normalize(processed_intensity)
|
||||
# Print summary of results
|
||||
printed_count = 0
|
||||
for info in calibration_info
|
||||
if !isempty(info) && printed_count < 5
|
||||
println(info)
|
||||
printed_count += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# --- Step 5: Peak Detection ---
|
||||
local pk_mz, pk_int
|
||||
pk_mz, pk_int = Float64[], Float64[] # Initialize empty
|
||||
if PEAK_DETECTION_PARAMS.enabled
|
||||
if PEAK_DETECTION_PARAMS.method == :profile
|
||||
pk_mz, pk_int = detect_peaks_profile(processed_mz, processed_intensity,
|
||||
half_window=PEAK_DETECTION_PARAMS.half_window,
|
||||
snr_threshold=PEAK_DETECTION_PARAMS.snr_threshold,
|
||||
min_peak_prominence=PEAK_DETECTION_PARAMS.min_peak_prominence,
|
||||
merge_peaks_tolerance=PEAK_DETECTION_PARAMS.merge_peaks_tolerance)
|
||||
elseif PEAK_DETECTION_PARAMS.method == :wavelet
|
||||
pk_mz, pk_int = detect_peaks_wavelet(processed_mz, processed_intensity)
|
||||
else # :centroid
|
||||
pk_mz, pk_int = detect_peaks_centroid(processed_mz, processed_intensity)
|
||||
end
|
||||
function apply_peak_alignment(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
print_step_header("Peak Alignment")
|
||||
|
||||
if !isempty(pk_mz)
|
||||
push!(all_processed_peaks_mz, pk_mz)
|
||||
push!(all_processed_peaks_int, pk_int)
|
||||
push!(spectrum_indices_with_peaks, idx)
|
||||
end
|
||||
end
|
||||
method = get(params, :method, :none)
|
||||
tolerance = get(params, :tolerance, 0.002)
|
||||
tolerance_unit = get(params, :tolerance_unit, :mz)
|
||||
println(" Method: $method, Tolerance: $tolerance $tolerance_unit")
|
||||
|
||||
# --- Visualization of a sample spectrum ---
|
||||
if i == 1 # Only plot the first processed spectrum
|
||||
println("\nGenerating example plots for spectrum #$idx...")
|
||||
fig = Figure(size=(1200, 800))
|
||||
|
||||
plot_spectrum_on_fig(fig[1,1], original_mz, original_intensity, "1. Original Spectrum")
|
||||
plot_spectrum_on_fig(fig[2,1], processed_mz, processed_intensity, "2. After All Steps (Before Peak Picking)")
|
||||
|
||||
# Plot detected peaks
|
||||
ax = Axis(fig[3,1], title="3. Detected Peaks")
|
||||
if !isempty(pk_mz)
|
||||
stem!(ax, pk_mz, pk_int)
|
||||
end
|
||||
|
||||
save(joinpath(RESULTS_DIR, "example_spectrum_processing.png"), fig)
|
||||
println("Saved example processing plots.")
|
||||
end
|
||||
end
|
||||
end
|
||||
println("\nIndividual spectrum processing complete.")
|
||||
|
||||
# --- Save QC Results ---
|
||||
if !isempty(qc_results)
|
||||
println("\n" * "="^20 * " Generating QC Report " * "="^20)
|
||||
CSV.write(joinpath(RESULTS_DIR, "qc_results.csv"), qc_results)
|
||||
println("QC results saved to qc_results.csv")
|
||||
|
||||
# Generate summary plots for QC
|
||||
fig = Figure(size=(1200, 600))
|
||||
|
||||
# PPM Error Histogram
|
||||
ppm_errors = filter(row -> row.metric == "ppm_error", qc_results).value
|
||||
if !isempty(ppm_errors)
|
||||
ax1 = Axis(fig[1,1], title="PPM Error Distribution", xlabel="PPM Error")
|
||||
hist!(ax1, ppm_errors, bins=30)
|
||||
end
|
||||
|
||||
# Resolution Histogram
|
||||
resolutions = filter(row -> row.metric == "resolution", qc_results).value
|
||||
if !isempty(resolutions)
|
||||
ax2 = Axis(fig[1,2], title="Resolution Distribution", xlabel="Resolution (FWHM)")
|
||||
hist!(ax2, resolutions, bins=30)
|
||||
end
|
||||
|
||||
save(joinpath(RESULTS_DIR, "qc_summary_plots.png"), fig)
|
||||
println("QC summary plots saved.")
|
||||
end
|
||||
|
||||
if isempty(all_processed_peaks_mz)
|
||||
@warn "No peaks were detected in any of the processed spectra. Skipping alignment and binning."
|
||||
if method == :none
|
||||
println(" - Skipping peak alignment")
|
||||
return
|
||||
end
|
||||
|
||||
# --- Step 6: Peak Alignment ---
|
||||
println("\n" * "="^20 * " Aligning Peaks " * "="^20)
|
||||
aligned_peaks_mz = copy(all_processed_peaks_mz)
|
||||
if PEAK_ALIGNMENT_PARAMS.enabled
|
||||
# Create a reference peak list (e.g., from the spectrum with the most peaks)
|
||||
ref_idx = argmax(length.(all_processed_peaks_mz))
|
||||
reference_peaks = all_processed_peaks_mz[ref_idx]
|
||||
println("Using spectrum $(spectrum_indices_with_peaks[ref_idx]) as alignment reference.")
|
||||
|
||||
for i in 1:length(aligned_peaks_mz)
|
||||
if i == ref_idx continue end
|
||||
alignment_func = align_peaks_lowess(reference_peaks, aligned_peaks_mz[i],
|
||||
tolerance=PEAK_ALIGNMENT_PARAMS.tolerance,
|
||||
tolerance_unit=PEAK_ALIGNMENT_PARAMS.tolerance_unit,
|
||||
min_matched_peaks=PEAK_ALIGNMENT_PARAMS.min_matched_peaks)
|
||||
|
||||
aligned_peaks_mz[i] = alignment_func(aligned_peaks_mz[i])
|
||||
end
|
||||
println("Peak alignment complete.")
|
||||
ref_find_idx = findfirst(s -> !isempty(s.peaks), spectra)
|
||||
if ref_find_idx === nothing
|
||||
println(" - Insufficient spectra with peaks for alignment. Skipping.")
|
||||
return
|
||||
end
|
||||
|
||||
# --- Step 7: Peak Binning & Feature Matrix Generation ---
|
||||
println("\n" * "="^20 * " Binning Peaks " * "="^20)
|
||||
if PEAK_BINNING_PARAMS.enabled
|
||||
feature_matrix, mz_bins = bin_peaks(aligned_peaks_mz, all_processed_peaks_int,
|
||||
PEAK_BINNING_PARAMS.tolerance,
|
||||
tolerance_unit=PEAK_BINNING_PARAMS.tolerance_unit,
|
||||
frequency_threshold=PEAK_BINNING_PARAMS.frequency_threshold)
|
||||
ref_spectrum = spectra[ref_find_idx]
|
||||
ref_peaks_mz = [p.mz for p in ref_spectrum.peaks]
|
||||
println(" - Using spectrum $(ref_spectrum.id) as reference with $(length(ref_peaks_mz)) peaks")
|
||||
|
||||
if !isempty(feature_matrix)
|
||||
df = DataFrame(feature_matrix, :auto)
|
||||
rename!(df, ["bin_$(i)" for i in 1:size(df, 2)])
|
||||
insertcols!(df, 1, :spectrum_idx => spectrum_indices_with_peaks)
|
||||
|
||||
CSV.write(joinpath(RESULTS_DIR, "feature_matrix.csv"), df)
|
||||
println("Feature matrix saved to feature_matrix.csv")
|
||||
|
||||
# Save bin m/z ranges
|
||||
bin_df = DataFrame(bin_index=1:length(mz_bins), mz_start=[b[1] for b in mz_bins], mz_end=[b[2] for b in mz_bins])
|
||||
CSV.write(joinpath(RESULTS_DIR, "bin_definitions.csv"), bin_df)
|
||||
println("Bin definitions saved to bin_definitions.csv")
|
||||
else
|
||||
@warn "Feature matrix was empty after binning."
|
||||
end
|
||||
Threads.@threads for s in spectra
|
||||
if s.id == ref_spectrum.id || isempty(s.peaks)
|
||||
continue
|
||||
end
|
||||
|
||||
println("\nPreprocessing Test Suite finished successfully!")
|
||||
current_peaks_mz = [p.mz for p in s.peaks]
|
||||
alignment_func = align_peaks_lowess(ref_peaks_mz, current_peaks_mz; method=method, tolerance=tolerance, tolerance_unit=tolerance_unit)
|
||||
|
||||
s.mz = alignment_func.(s.mz) # Update m/z axis
|
||||
|
||||
# Update peak m/z values
|
||||
for i in 1:length(s.peaks)
|
||||
old_peak = s.peaks[i]
|
||||
aligned_peak_mz = alignment_func(old_peak.mz)
|
||||
s.peaks[i] = (mz=aligned_peak_mz, intensity=old_peak.intensity, fwhm=old_peak.fwhm, shape_r2=old_peak.shape_r2, snr=old_peak.snr, prominence=old_peak.prominence)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function apply_normalization(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
print_step_header("Normalization")
|
||||
|
||||
# --- Execute ---
|
||||
run_preprocessing_suite()
|
||||
method = get(params, :method, :tic)
|
||||
println(" Method: $method")
|
||||
|
||||
# Safely plot the first spectrum
|
||||
if !isempty(spectra)
|
||||
s1 = findfirst(s -> s.id == 1, spectra)
|
||||
if s1 !== nothing
|
||||
s = spectra[s1]
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
normalized_intensity = MSI_src.apply_normalization(s.intensity; method=method)
|
||||
plot_spectrum_step(s.mz, normalized_intensity, "normalization")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Threads.@threads for s in spectra
|
||||
if validate_spectrum(s.mz, s.intensity)
|
||||
s.intensity = MSI_src.apply_normalization(s.intensity; method=method)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function apply_peak_binning(spectra::Vector{MutableSpectrum}, params::Dict)
|
||||
print_step_header("Peak Binning")
|
||||
|
||||
method = get(params, :method, :adaptive)
|
||||
tolerance = get(params, :tolerance, 20.0)
|
||||
tolerance_unit = get(params, :tolerance_unit, :ppm)
|
||||
min_peak_per_bin = get(params, :min_peak_per_bin, 3)
|
||||
max_bin_width_ppm = get(params, :max_bin_width_ppm, 150.0)
|
||||
intensity_weighted_centers = get(params, :intensity_weighted_centers, true)
|
||||
println(" Method: $method, Tolerance: $tolerance $tolerance_unit")
|
||||
|
||||
if isempty(spectra) || all(s -> isempty(s.peaks), spectra)
|
||||
@warn "No peaks found for binning. Returning empty feature matrix."
|
||||
return nothing, nothing
|
||||
end
|
||||
|
||||
println(" - Binning peaks from $(length(spectra)) spectra")
|
||||
|
||||
binning_params = PeakBinningParams(
|
||||
method=method,
|
||||
tolerance=tolerance,
|
||||
tolerance_unit=tolerance_unit,
|
||||
min_peak_per_bin=min_peak_per_bin,
|
||||
max_bin_width_ppm=max_bin_width_ppm,
|
||||
intensity_weighted_centers=intensity_weighted_centers
|
||||
)
|
||||
|
||||
feature_matrix, bin_definitions = bin_peaks(spectra, binning_params)
|
||||
|
||||
if feature_matrix !== nothing && bin_definitions !== nothing
|
||||
println(" - Generated feature matrix: $(size(feature_matrix.matrix))")
|
||||
println(" - Number of bins: $(length(bin_definitions))")
|
||||
end
|
||||
|
||||
return feature_matrix, bin_definitions
|
||||
end
|
||||
|
||||
function save_feature_matrix(feature_matrix, bin_definitions)
|
||||
print_step_header("Saving Results")
|
||||
|
||||
# Save feature matrix as CSV
|
||||
csv_path = joinpath(OUTPUT_DIR, "feature_matrix.csv")
|
||||
|
||||
# Create column headers
|
||||
bin_headers = ["bin_$(i)_$(round(def[1], digits=4))-$(round(def[2], digits=4))"
|
||||
for (i, def) in enumerate(bin_definitions)]
|
||||
|
||||
# Open file for writing
|
||||
open(csv_path, "w") do io
|
||||
# Write header row
|
||||
write(io, "spectrum_index," * join(bin_headers, ",") * "\n")
|
||||
|
||||
# Write data rows
|
||||
for r_idx in 1:size(feature_matrix.matrix, 1)
|
||||
write(io, "$(feature_matrix.sample_ids[r_idx]),")
|
||||
for c_idx in 1:size(feature_matrix.matrix, 2)
|
||||
write(io, "$(feature_matrix.matrix[r_idx, c_idx])")
|
||||
if c_idx < size(feature_matrix.matrix, 2)
|
||||
write(io, ",")
|
||||
end
|
||||
end
|
||||
write(io, "\n")
|
||||
end
|
||||
end
|
||||
println(" - Saved feature matrix: $csv_path")
|
||||
|
||||
# Save bin definitions (still using DataFrame as it's small)
|
||||
bins_path = joinpath(OUTPUT_DIR, "bin_definitions.csv")
|
||||
bins_df = DataFrame(
|
||||
bin_index = 1:length(bin_definitions),
|
||||
mz_start = [def[1] for def in bin_definitions],
|
||||
mz_end = [def[2] for def in bin_definitions],
|
||||
mz_center = [(def[1] + def[2])/2 for def in bin_definitions]
|
||||
)
|
||||
CSV.write(bins_path, bins_df)
|
||||
println(" - Saved bin definitions: $bins_path")
|
||||
|
||||
return csv_path, bins_path
|
||||
end
|
||||
|
||||
# ===================================================================
|
||||
# USER OVERRIDES: Manually specify parameters here
|
||||
# ===================================================================
|
||||
# This dictionary allows you to override any auto-detected parameters.
|
||||
# The structure should match the output of `main_precalculation`.
|
||||
# Example: Force a less aggressive peak prominence threshold.
|
||||
const USER_OVERRIDES = Dict(
|
||||
:PeakPicking => Dict(
|
||||
#:snr_threshold => 1.5,
|
||||
:half_window => 5,
|
||||
:snr_threshold => 15.0,
|
||||
#:merge_peaks_tolerance => 2.5,
|
||||
#:half_window => 2
|
||||
),
|
||||
# :Smoothing => Dict(
|
||||
# :window => 11
|
||||
# )
|
||||
#:PeakAlignment => Dict(
|
||||
# :tolerance => 0.01
|
||||
#)
|
||||
:PeakBinningParams => Dict(
|
||||
:tolerance => 30.0
|
||||
)
|
||||
)
|
||||
|
||||
# ===================================================================
|
||||
# MAIN PREPROCESSING PIPELINE
|
||||
# ===================================================================
|
||||
|
||||
function run_preprocessing_pipeline()
|
||||
println("="^80)
|
||||
println("MSI PREPROCESSING PIPELINE")
|
||||
println("="^80)
|
||||
|
||||
# Ensure output directory exists
|
||||
ensure_output_dir()
|
||||
|
||||
# Load data
|
||||
println("\nLoading data: $(basename(TEST_FILE))")
|
||||
msi_data = OpenMSIData(TEST_FILE)
|
||||
|
||||
# Precompute analytics
|
||||
println("\nPrecomputing analytics...")
|
||||
precompute_analytics(msi_data)
|
||||
|
||||
# Run precalculation to get automatic parameters
|
||||
println("\nRunning precalculation for automatic parameter determination...")
|
||||
auto_params = main_precalculation(msi_data, reference_peaks=reference_peaks, mask_path=MASK_ROUTE)
|
||||
|
||||
# --- Apply User Overrides ---
|
||||
if !isempty(USER_OVERRIDES)
|
||||
println("\nApplying user overrides...")
|
||||
for (step, params) in USER_OVERRIDES
|
||||
if haskey(auto_params, step)
|
||||
for (param, value) in params
|
||||
@info "OVERRIDE: For step `:$step`, setting `:$param` to `$value` (was `$(get(auto_params[step], param, "not set"))`)"
|
||||
end
|
||||
merge!(auto_params[step], params)
|
||||
else
|
||||
@warn "User override for non-existent step `:$step` ignored."
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# --- Final Parameter Summary ---
|
||||
println("\n" * "-"^60)
|
||||
println("FINAL PARAMETERS FOR PIPELINE RUN")
|
||||
println("-"^60)
|
||||
for (step, params) in sort(collect(pairs(auto_params)), by=p -> p.first)
|
||||
println(" - Step: $step")
|
||||
if isempty(params)
|
||||
println(" (No parameters)")
|
||||
continue
|
||||
end
|
||||
for (param, value) in sort(collect(pairs(params)), by=p -> p.first)
|
||||
println(" - $(rpad(param, 25)): $value")
|
||||
end
|
||||
end
|
||||
|
||||
# Define pipeline steps
|
||||
pipeline_steps = [
|
||||
"baseline_correction",
|
||||
"smoothing",
|
||||
"peak_picking",
|
||||
"calibration",
|
||||
"peak_alignment",
|
||||
"normalization",
|
||||
"peak_binning"
|
||||
]
|
||||
|
||||
println("\nPipeline steps: $(join(pipeline_steps, " -> "))")
|
||||
|
||||
# Initialize `current_spectra` as a Vector of mutable structs for in-place modification
|
||||
println("\nInitializing spectra data structure...")
|
||||
num_spectra = length(msi_data.spectra_metadata)
|
||||
current_spectra = Vector{MutableSpectrum}(undef, num_spectra)
|
||||
_iterate_spectra_fast(msi_data) do idx, mz, intensity
|
||||
current_spectra[idx] = MutableSpectrum(idx, mz, intensity, [])
|
||||
|
||||
# Plot raw spectrum without masks
|
||||
if idx == 1
|
||||
plot_spectrum_step(mz, intensity, "raw_unmasked_spectrum")
|
||||
end
|
||||
end
|
||||
|
||||
# These variables will be populated by the pipeline steps
|
||||
feature_matrix = nothing
|
||||
bin_definitions = nothing
|
||||
|
||||
# Apply pipeline steps, modifying `current_spectra` in-place
|
||||
for step in pipeline_steps
|
||||
println("\n" * "-"^60)
|
||||
println("PROCESSING STEP: $step")
|
||||
println("-"^60)
|
||||
|
||||
if step == "baseline_correction"
|
||||
@time apply_baseline_correction(current_spectra, auto_params[:BaselineCorrection], msi_data)
|
||||
|
||||
elseif step == "smoothing"
|
||||
apply_smoothing(current_spectra, auto_params[:Smoothing])
|
||||
|
||||
elseif step == "peak_picking"
|
||||
@time apply_peak_picking(current_spectra, auto_params[:PeakPicking])
|
||||
|
||||
elseif step == "calibration"
|
||||
@time apply_calibration(current_spectra, auto_params[:Calibration], reference_peaks)
|
||||
|
||||
elseif step == "peak_alignment"
|
||||
@time apply_peak_alignment(current_spectra, auto_params[:PeakAlignment])
|
||||
|
||||
elseif step == "normalization"
|
||||
@time apply_normalization(current_spectra, auto_params[:Normalization])
|
||||
|
||||
elseif step == "peak_binning"
|
||||
# This step is different as it generates the final matrix, not modifying spectra in-place
|
||||
feature_matrix, bin_definitions = @time apply_peak_binning(current_spectra, auto_params[:PeakBinningParams])
|
||||
|
||||
if feature_matrix !== nothing
|
||||
@time save_feature_matrix(feature_matrix, bin_definitions)
|
||||
end
|
||||
|
||||
else
|
||||
@warn "Unknown step: $step, skipping"
|
||||
end
|
||||
|
||||
# Print progress
|
||||
println("✓ Completed step: $step")
|
||||
end
|
||||
|
||||
# Clean up
|
||||
close(msi_data)
|
||||
GC.gc()
|
||||
|
||||
println("\n" * "="^80)
|
||||
println("PREPROCESSING PIPELINE COMPLETED SUCCESSFULLY!")
|
||||
println("Results saved to: $OUTPUT_DIR")
|
||||
println("="^80)
|
||||
end
|
||||
|
||||
# ===================================================================
|
||||
# EXECUTE PIPELINE
|
||||
# ===================================================================
|
||||
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
@time run_preprocessing_pipeline()
|
||||
end
|
||||
@ -75,9 +75,9 @@ const COORDS_TO_PLOT = (50, 50) # Example coordinates (X, Y)
|
||||
# --- Output Directory ---
|
||||
const RESULTS_DIR = "test/results"
|
||||
|
||||
test1 = false
|
||||
test2 = true
|
||||
test3 = false
|
||||
test1 = true
|
||||
test2 = false
|
||||
test3 = true
|
||||
|
||||
# ===================================================================
|
||||
# DATA VALIDATION UTILITY
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user