""" This file contains the workflow for converting .mzML files (with one spectrum per pixel) into a proper .imzML/.ibd file pair, using a separate synchronization file. It replicates the functionality of the original R scripts that use MALDIquant. """ using DataFrames, Printf, CSV # This file assumes that the main application file (e.g., app.jl) has already included # the necessary source files: MSIData.jl, mzML.jl, imzML.jl # A struct to hold the processed pixel data before exporting """ ProcessedPixel A temporary struct to hold the data for a single, fully rendered pixel before it is written to the final `.ibd` file. # Fields - `coords`: A tuple `(x, y)` of the pixel's spatial coordinates. - `mz`: The m/z array for the pixel. - `intensity`: The calculated intensity array for the pixel. """ struct ProcessedPixel coords::Tuple{Int, Int} mz::Vector{Float64} # Assuming m/z is consistent, can be optimized later intensity::Vector{Float32} end """ BinaryMetadata A struct to hold the file offset and length for a spectrum's binary data arrays (m/z and intensity) after they have been written to the `.ibd` file. # Fields - `mz_offset`, `mz_length`: Byte offset and length for the m/z array. - `int_offset`, `int_length`: Byte offset and length for the intensity array. """ struct BinaryMetadata mz_offset::UInt64 mz_length::UInt64 int_offset::UInt64 int_length::UInt64 end """ GetMzmlScanTime_linebyline(fileName::String) Parses a `.mzML` file line-by-line to extract the scan start time for each spectrum. This is a slow and memory-intensive fallback method used only when the faster, index-based `GetMzmlScanTime` fails. # Arguments - `fileName`: Path to the `.mzML` file. # Returns - A `Matrix{Int64}` where each row is `[spectrum_index, time_in_milliseconds]`. """ function GetMzmlScanTime_linebyline(fileName::String) times = Tuple{Int64, Int64}[] try file_size = filesize(fileName) estimated_spectra = max(1000, file_size ÷ 10000) sizehint!(times, estimated_spectra) catch end open(fileName, "r") do stream state = :outside_spectrum current_index = 0 current_time = nothing for line in eachline(stream) if state == :outside_spectrum if occursin("", line) state = :outside_spectrum if current_index > 0 && current_time !== nothing push!(times, (current_index, current_time)) end end end end end result = Matrix{Int64}(undef, length(times), 2) for (i, t) in enumerate(times) result[i, 1] = t[1] result[i, 2] = t[2] end return result end """ GetMzmlScanTime(fileName::String) Parses a .mzML file to extract the scan start time for each spectrum. Uses the indexed part of the .mzML file for fast access, falling back to a slower line-by-line parse if the index is not present. # Arguments * `fileName`: Path to the .mzML file. # Returns - A `Matrix{Int64}` where each row is `[spectrum_index, time_in_milliseconds]`. """ function GetMzmlScanTime(fileName::String) times = Tuple{Int64, Int64}[] try open(fileName, "r") do stream # 1. Find and parse the spectrum index offsets seekend(stream) end_chunk_size = min(filesize(stream), 8192) seek(stream, filesize(stream) - end_chunk_size) footer = read(stream, String) index_offset_match = match(r"(\d+)", footer) if index_offset_match === nothing @warn "No found. Falling back to slow line-by-line parsing for scan times. This may be memory intensive." return GetMzmlScanTime_linebyline(fileName) end index_offset = parse(Int64, index_offset_match.captures[1]) seek(stream, index_offset) # The find_tag function is defined in ParserHelpers.jl if find_tag(stream, r"", line) || occursin("", line) break end end if current_time !== nothing push!(times, (idx, current_time)) end end end catch e @error "Failed to parse scan times with indexed method. Falling back to line-by-line." exception=(e, catch_backtrace()) return GetMzmlScanTime_linebyline(fileName) end # The index is already sorted by spectrum index, so no need to sort `times`. # Normalize times relative to the first scan if !isempty(times) first_time = times[1][2] for i in eachindex(times) times[i] = (times[i][1], times[i][2] - first_time) end end result = Matrix{Int64}(undef, length(times), 2) for (i, t) in enumerate(times) result[i, 1] = t[1] result[i, 2] = t[2] end return result end """ MatchAcquireTime(sync_file_path::String, scans::Matrix{Int64}) Correlates pixel acquisition times from a synchronization file with scan acquisition times from an mzML file. This is a Julia implementation of the `MatchAcquireTime` function from the R scripts. # Arguments * `sync_file_path`: Path to the synchronization file (.txt). * `scans`: A matrix of scan times, as returned by `GetMzmlScanTime`. # Returns - A `Matrix{Int64}` where each row is `[pixel_index, pixel_time_ms, first_scan_index, last_scan_index]`. """ function MatchAcquireTime(sync_file_path::String, scans::Matrix{Int64}; img_width::Int=0, img_height::Int=0) if !isfile(sync_file_path) error("Synchronization file not found: $sync_file_path") end pixel_df = CSV.read(sync_file_path, DataFrame, header=false, skipto=3) pixel_matrix = Matrix(pixel_df) num_pixels_original = size(pixel_matrix, 1) if num_pixels_original == 0 return zeros(Int64, 0, 5) end # Determine image dimensions and generate coordinates current_img_width = img_width current_img_height = img_height if current_img_width == 0 || current_img_height == 0 if size(pixel_matrix, 2) >= 3 @info "Sync file contains X, Y coordinates" coordinates = convert(Matrix{Int}, pixel_matrix[:, 1:2]) pixel_times = convert(Vector{Int64}, pixel_matrix[:, 3]) current_img_width = maximum(coordinates[:, 1]) current_img_height = maximum(coordinates[:, 2]) # Apply R's pixel truncation logic num_pixels = (floor(Int, (num_pixels_original - 1) / current_img_width)) * current_img_width if num_pixels <= 0 error("Calculated num_pixels to process is zero or negative: $num_pixels") end # Truncate arrays final_coordinates = coordinates[1:num_pixels, :] final_pixel_times = pixel_times[1:num_pixels] else @info "Sync file contains Index and Time; generating coordinates" pixel_times = convert(Vector{Int64}, pixel_matrix[:, 2]) # R's width detection logic diffs = pixel_matrix[2:end, 1] .- pixel_matrix[1:end-1, 1] width_indices = findall(x -> x != 1, diffs) if !isempty(width_indices) current_img_width = width_indices[1] else current_img_width = num_pixels_original end current_img_height = num_pixels_original ÷ current_img_width # R's pixel truncation logic num_pixels = (floor(Int, (num_pixels_original - 1) / current_img_width)) * current_img_width if num_pixels <= 0 error("Calculated num_pixels to process is zero or negative: $num_pixels") end # Generate coordinates for truncated pixels final_coordinates = zeros(Int, num_pixels, 2) final_pixel_times = zeros(Int64, num_pixels) for i in 1:num_pixels idx = pixel_matrix[i, 1] final_coordinates[i, 1] = ((idx - 1) % current_img_width) + 1 final_coordinates[i, 2] = fld(idx - 1, current_img_width) + 1 final_pixel_times[i] = pixel_times[i] end end else @info "Using provided dimensions: $(current_img_width)x$(current_img_height)" pixel_times = convert(Vector{Int64}, pixel_matrix[:, 2]) # R's pixel truncation logic (FIXED: consistent formula) num_pixels = (floor(Int, (num_pixels_original - 1) / current_img_width)) * current_img_width if num_pixels <= 0 error("Calculated num_pixels to process is zero or negative: $num_pixels") end # Generate coordinates final_coordinates = zeros(Int, num_pixels, 2) final_pixel_times = zeros(Int64, num_pixels) for i in 1:num_pixels idx = pixel_matrix[i, 1] final_coordinates[i, 1] = ((idx - 1) % current_img_width) + 1 final_coordinates[i, 2] = fld(idx - 1, current_img_width) + 1 final_pixel_times[i] = pixel_times[i] end end # Normalize pixel times if !isempty(final_pixel_times) min_pixel_time = minimum(final_pixel_times) final_pixel_times .-= min_pixel_time end num_scans = size(scans, 1) if num_scans == 0 return hcat(final_coordinates, final_pixel_times, zeros(Int64, num_pixels, 2)) end # FIXED R's time matching algorithm with bounds checking first_idx = 1 last_idx = 1 index_matrix = zeros(Int, num_pixels, 2) for i_pixel in 1:num_pixels pixel_time = final_pixel_times[i_pixel] # Find the first scan that reaches or exceeds pixel time while last_idx <= num_scans && scans[last_idx, 2] < pixel_time last_idx += 1 end # Ensure valid indices if last_idx > num_scans # No more scans available for remaining pixels index_matrix[i_pixel:end, 1] .= num_scans + 1 # Invalid index index_matrix[i_pixel:end, 2] .= num_scans # Invalid index break end # Assign scan indices with bounds checking start_scan = max(1, first_idx) end_scan = max(1, last_idx - 1) # Ensure start_scan <= end_scan if start_scan > end_scan start_scan = end_scan end index_matrix[i_pixel, 1] = start_scan index_matrix[i_pixel, 2] = end_scan # Update for next iteration (R's algorithm) first_idx = last_idx - 1 last_idx = first_idx # Ensure first_idx doesn't go below 1 if first_idx < 1 first_idx = 1 last_idx = 1 end end return hcat(final_coordinates, final_pixel_times, index_matrix) end """ RenderPixel(pixel_info, scans, msi_data, scan_time_deltas, pixel_time_deltas) Reconstructs the spectrum for a single pixel by combining intensities from the raw MS scans that occurred during the pixel's acquisition time. It uses a weighted interpolation scheme based on the relative timing of scans and pixels. This function handles three cases: 1. A single scan falls entirely within the pixel's time window. 2. The pixel's time window is covered by two partial scans. 3. The pixel's time window covers one or more full scans plus two partial scans. # Arguments - `pixel_info`: A row from the timing matrix containing the pixel's time and scan indices. - `scans`: The matrix of scan times. - `msi_data`: The `MSIData` object for the source `.mzML` file. - `scan_time_deltas`: Pre-calculated time durations for each scan. - `pixel_time_deltas`: Pre-calculated time durations for each pixel. # Returns - A tuple `(mz_array, intensity_array)` for the rendered pixel spectrum. """ function RenderPixel( pixel_info::AbstractVector{Int64}, scans::AbstractMatrix{Int64}, msi_data::MSIData, scan_time_deltas::AbstractVector{Int64}, pixel_time_deltas::AbstractVector{Int64} ) pixel_time = pixel_info[3] first_scan = pixel_info[4] last_scan = pixel_info[5] num_actions = last_scan - first_scan # Get reference m/z array from the first scan involved. # This call remains type-unstable, but its impact is now isolated and only paid once. mz_array, _ = GetSpectrum(msi_data, first_scan) # If the first spectrum was empty, we can't do anything else. if isempty(mz_array) return (mz_array, Float32[]) end new_intensity = zeros(Float32, length(mz_array)) # SAFETY: Ensure we have valid scan indices if first_scan < 1 || last_scan > size(scans, 1) || first_scan > last_scan return (mz_array, new_intensity) # Return zero intensity for invalid ranges end if num_actions == 0 # Single scan contributes to the pixel process_spectrum(msi_data, first_scan) do _, intensity # SAFETY: Ensure positive scaling scale = max(pixel_time_deltas[first_scan] / scan_time_deltas[first_scan], 0.0f0) new_intensity .= intensity .* scale end elseif num_actions == 1 # Two partial scans contribute # First partial scan process_spectrum(msi_data, first_scan) do _, intensity1 scale1 = max((scans[last_scan, 2] - pixel_time) / scan_time_deltas[first_scan], 0.0f0) new_intensity .+= intensity1 .* scale1 end # Second partial scan process_spectrum(msi_data, last_scan) do _, intensity2 next_pixel_time = pixel_time + pixel_time_deltas[first_scan] scale2 = max((next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan], 0.0f0) new_intensity .+= intensity2 .* scale2 end elseif num_actions > 1 # Multiple scans contribute # First partial scan process_spectrum(msi_data, first_scan) do _, intensity1 scale1 = max((scans[first_scan + 1, 2] - pixel_time) / scan_time_deltas[first_scan], 0.0f0) new_intensity .+= intensity1 .* scale1 end # Full scans in the middle for i in (first_scan + 1):(last_scan - 1) process_spectrum(msi_data, i) do _, intensity_middle new_intensity .+= intensity_middle end end # Last partial scan process_spectrum(msi_data, last_scan) do _, intensity2 next_pixel_time = pixel_time + pixel_time_deltas[first_scan] scale2 = max((next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan], 0.0f0) new_intensity .+= intensity2 .* scale2 end end # FINAL SAFETY: Clamp any negative values to zero new_intensity = max.(new_intensity, 0.0f0) return (mz_array, new_intensity) end """ ConvertMzmlToImzml(source_file, target_ibd_file, timing_matrix, scans) Orchestrates the conversion of spectra from a `.mzML` file into a binary `.ibd` file. It iterates through each pixel defined in the `timing_matrix`, calls `RenderPixel` to reconstruct the pixel's spectrum, and writes the resulting m/z and intensity arrays to the `.ibd` file in little-endian byte order. # Arguments - `source_file`: Path to the source `.mzML` file. - `target_ibd_file`: Path for the output `.ibd` binary file. - `timing_matrix`: The output from `MatchAcquireTime`, mapping pixels to scans. - `scans`: The matrix of scan times from `GetMzmlScanTime`. # Returns - A tuple `(binary_meta_vec, coords_vec, (width, height))` containing: - A vector of `BinaryMetadata` for each spectrum. - A vector of `(x, y)` coordinate tuples. - A tuple of the final image dimensions. """ function ConvertMzmlToImzml(source_file::String, target_ibd_file::String, timing_matrix::Matrix{Int64}, scans::Matrix{Int64}) if size(timing_matrix, 1) == 0 # Create an empty .ibd file if there's nothing to process open(target_ibd_file, "w") do ibd_stream write(ibd_stream, zeros(UInt8, 16)) # UUID placeholder end return BinaryMetadata[], Tuple{Int, Int}[], (0, 0) end width = maximum(timing_matrix[:, 1]) height = maximum(timing_matrix[:, 2]) msi_data = OpenMSIData(source_file) scan_time_deltas = zeros(Int64, size(scans, 1)) if size(scans, 1) > 1 for i in 1:(size(scans, 1) - 1) delta = scans[i+1, 2] - scans[i, 2] scan_time_deltas[i] = max(1, delta) end scan_time_deltas[end] = max(1, scan_time_deltas[end-1]) end pixel_time_deltas = zeros(Int64, size(timing_matrix, 1)) if size(timing_matrix, 1) > 1 for i in 1:(size(timing_matrix, 1) - 1) delta = timing_matrix[i+1, 3] - timing_matrix[i, 3] pixel_time_deltas[i] = max(1, delta) end pixel_time_deltas[end] = max(1, pixel_time_deltas[end-1]) end binary_meta_vec = BinaryMetadata[] sizehint!(binary_meta_vec, size(timing_matrix, 1)) coords_vec = Tuple{Int, Int}[] sizehint!(coords_vec, size(timing_matrix, 1)) empty_pixel_count = 0 open(target_ibd_file, "w") do ibd_stream write(ibd_stream, zeros(UInt8, 16)) # UUID placeholder for i in 1:size(timing_matrix, 1) pixel_info = timing_matrix[i, :] x, y = pixel_info[1], pixel_info[2] push!(coords_vec, (x, y)) first_scan = pixel_info[4] last_scan = pixel_info[5] if first_scan > last_scan || first_scan < 1 || last_scan > size(scans, 1) empty_pixel_count += 1 # For empty pixels, offsets point to the current end of file, with zero length current_pos = position(ibd_stream) push!(binary_meta_vec, BinaryMetadata(current_pos, 0, current_pos, 0)) continue end mz, intensity = RenderPixel(pixel_info, scans, msi_data, scan_time_deltas, pixel_time_deltas) # Write m/z array mz_offset = position(ibd_stream) for val in mz write(ibd_stream, htol(Float64(val))) end mz_length = position(ibd_stream) - mz_offset # Write intensity array int_offset = position(ibd_stream) for val in intensity write(ibd_stream, htol(Float32(val))) end int_length = position(ibd_stream) - int_offset push!(binary_meta_vec, BinaryMetadata(mz_offset, mz_length, int_offset, int_length)) end end @info "Found and processed $empty_pixel_count empty pixels out of $(size(timing_matrix, 1)) total." return binary_meta_vec, coords_vec, (width, height) end """ ExportImzml(target_file, binary_meta, coords, dims) Generates the `.imzML` metadata file. This XML file contains all the necessary metadata to interpret the corresponding `.ibd` binary file, including references to external data offsets, image dimensions, and CV parameters describing the experiment and data format. # Arguments - `target_file`: The path for the output `.imzML` file. - `binary_meta`: A vector of `BinaryMetadata` structs with offset and length info. - `coords`: A vector of `(x, y)` coordinates for each spectrum. - `dims`: A tuple `(width, height)` of the final image dimensions. # Returns - `true` on success, `false` on failure. """ function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, coords::Vector{Tuple{Int, Int}}, dims::Tuple{Int, Int}) ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") if isempty(binary_meta) @warn "No binary metadata to export; creating empty imzML file." # Still create a valid, empty imzML file end try # The .ibd file is now written by ConvertMzmlToImzml. # This function is only responsible for the .imzML XML metadata file. open(target_file, "w") do imzml_stream # XML Header write(imzml_stream, """ """) # CV List, File Description, etc. (static parts) write(imzml_stream, """ """) write(imzml_stream, """ """) write(imzml_stream, """ """) write(imzml_stream, """ """) write(imzml_stream, """ """) write(imzml_stream, """ """) write(imzml_stream, """ """) write(imzml_stream, """ """) # Run and Spectrum List spectrum_offsets = UInt64[] write(imzml_stream, """ """) # Write each spectrum's metadata for (i, meta) in enumerate(binary_meta) x, y = coords[i] spectrum_start = position(imzml_stream) push!(spectrum_offsets, spectrum_start) # Calculate number of points from byte length mz_points = meta.mz_length ÷ sizeof(Float64) int_points = meta.int_length ÷ sizeof(Float32) write(imzml_stream, """ """) end write(imzml_stream, """ """) # Index List index_list_start = position(imzml_stream) write(imzml_stream, """ """) for (i, offset) in enumerate(spectrum_offsets) write(imzml_stream, " $offset\n") end write(imzml_stream, """ $index_list_start """) end println("Successfully created: $target_file") println("Successfully created: $ibd_file") return true catch e @error "Failed to export imzML metadata file" exception=(e, catch_backtrace()) # Clean up partial .imzML file isfile(target_file) && rm(target_file, force=true) # Do not delete the .ibd file as it might be useful for debugging return false end end """ ImportMzmlFile(source_file::String, sync_file::String, target_file::String) Main workflow function to convert a .mzML file to an .imzML file. # Arguments * `source_file`: Path to the input .mzML file. * `sync_file`: Path to the synchronization text file. * `target_file`: Path for the output .imzML file (the .ibd will be named accordingly). * `img_width`: width dimention for the creation of the x axis * `img_height`: height dimention for the creation of the y axis """ function ImportMzmlFile(source_file::String, sync_file::String, target_file::String; img_width::Int=0, img_height::Int=0) println("Step 1: Getting scan times from .mzML file...") scans = GetMzmlScanTime(source_file) println("Step 2: Matching acquisition times...") timing_matrix = MatchAcquireTime(sync_file, scans; img_width=img_width, img_height=img_height) println("Step 3: Converting spectra and writing .ibd file...") ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") binary_meta, coords, (width, height) = ConvertMzmlToImzml(source_file, ibd_file, timing_matrix, scans) # Flip image vertically to match R script output flipped_coords = [(x, height - y + 1) for (x, y) in coords] println("Step 4: Exporting .imzML metadata file...") success = ExportImzml(target_file, binary_meta, flipped_coords, (width, height)) if success println("Conversion successful: $target_file") else println("Conversion failed.") end return success end