From b1d7838a6002c3f9293c30a54a039306ec6023c8 Mon Sep 17 00:00:00 2001 From: Pixelguy14 Date: Thu, 25 Sep 2025 10:42:33 -0600 Subject: [PATCH 01/19] Implemented imzml and mzml lazy loading, imzml to load spectra matrix, working on mzml to imzml --- .gitignore | 1 + app.jl | 33 +- config/env/global.jl | 3 +- julia_imzML_visual.jl | 2 +- "ongoing_tasks.txt\n" | 27 -- scripts/build.jl | 23 ++ scripts/precompile.jl | 45 +++ src/Bitmap.jl | 399 ++++++++++++++++++++++++ src/Common.jl | 94 ++++++ src/DataAccess.jl | 82 +++++ src/MzmlConverter.jl | 270 ++++++++++++++++ src/ParserHelpers.jl | 122 ++++++++ src/imzML.jl | 699 ++++++++++++++++++++++++++++++++++++++++++ src/imzML_old.jl | 246 +++++++++++++++ src/mzML.jl | 191 ++++++++++++ src/mzML_old.jl | 350 +++++++++++++++++++++ start_MSI_GUI.jl | 6 +- 17 files changed, 2547 insertions(+), 46 deletions(-) delete mode 100644 "ongoing_tasks.txt\n" create mode 100644 scripts/build.jl create mode 100644 scripts/precompile.jl create mode 100644 src/Bitmap.jl create mode 100644 src/Common.jl create mode 100644 src/DataAccess.jl create mode 100644 src/MzmlConverter.jl create mode 100644 src/ParserHelpers.jl create mode 100644 src/imzML.jl create mode 100644 src/imzML_old.jl create mode 100644 src/mzML.jl create mode 100644 src/mzML_old.jl diff --git a/.gitignore b/.gitignore index 6832527..1905238 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ public/css/imgOver.png !public/css/autogenerated.css !public/css/LABI_logo.png log/* +R_original_scripts/ diff --git a/app.jl b/app.jl index aaa9767..4bf8124 100644 --- a/app.jl +++ b/app.jl @@ -6,7 +6,7 @@ using Libz using PlotlyBase using CairoMakie using Colors -using julia_mzML_imzML +# using julia_mzML_imzML using Statistics using NaturalSort using Images @@ -15,6 +15,7 @@ using NativeFileDialog # Opens the file explorer depending on the OS using StipplePlotly using Base.Filesystem: mv # To rename files in the system include("./julia_imzML_visual.jl") +include("./src/imzML.jl") @genietools # == Reactive code == @@ -151,7 +152,7 @@ include("./julia_imzML_visual.jl") ), margin=attr(l=0,r=0,t=0,b=0,pad=0) ) - traceImg=PlotlyBase.heatmap(x=[], y=[]) + traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) @out plotdataImg=[traceImg] @out plotlayoutImg=layoutImg # For the image in the comparative view @@ -180,14 +181,14 @@ include("./julia_imzML_visual.jl") margin=attr(l=0,r=0,t=120,b=0,pad=0) ) # Dummy 2D scatter plot - traceSpectra=PlotlyBase.scatter(x=[], y=[], mode="lines") + traceSpectra=PlotlyBase.stem(x=Vector{Float64}(), y=Vector{Float64}(),marker=attr(size=1, color="blue", opacity=0.1)) # Create conection to frontend @out plotdata=[traceSpectra] @out plotlayout=layoutSpectra @in xCoord=0 @in yCoord=0 - @out xSpectraMz=Float64[] - @out ySpectraMz=Float64[] + @out xSpectraMz = Vector{Float64}() + @out ySpectraMz = Vector{Float64}() # Interactive plot reactions @in data_click=Dict{String,Any}() @@ -207,7 +208,7 @@ include("./julia_imzML_visual.jl") margin=attr(l=0,r=0,t=100,b=0,pad=0) ) # Dummy 2D surface plot - traceContour=PlotlyBase.scatter(x=[], y=[], mode="lines") + traceContour=PlotlyBase.scatter(x=Vector{Float64}(), y=Vector{Float64}(), mode="lines") # Create conection to frontend @out plotdataC=[traceContour] @out plotlayoutC=layoutContour @@ -233,7 +234,7 @@ include("./julia_imzML_visual.jl") x=1:10 y=1:10 z=[sin(i * j / 10) for i in x, j in y] - trace3D=PlotlyBase.surface(x=[], y=[], z=[], + trace3D=PlotlyBase.surface(x=Vector{Float64}(), y=Vector{Float64}(), z=Matrix{Float64}(undef, 0, 0), contours_z=attr( show=true, usecolormap=true, @@ -362,7 +363,7 @@ include("./julia_imzML_visual.jl") if isfile(full_route) && Nmass > 0 && Tol > 0 && Tol <=1 && colorLevel > 1 && colorLevel < 257 msg="File exists, Nmass=$(Nmass) Tol=$(Tol). Loading file will begin, please be patient." try - spectra=LoadImzml(full_route) + spectra=load_imzml(full_route) msg="File loaded. Creating spectra with the specific mass and tolerance, please be patient." slice=GetMzSliceJl(spectra,Nmass,Tol) fig=CairoMakie.Figure(size=(150, 250)) # Container @@ -588,7 +589,7 @@ include("./julia_imzML_visual.jl") plotdataImg, plotlayoutImg, imgWidth, imgHeight=loadImgPlot(imgInt) btnOpticalDisable=false else - traceImg=PlotlyBase.heatmap(x=[], y=[]) + traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImg=[traceImg] msgimg="" end @@ -615,7 +616,7 @@ include("./julia_imzML_visual.jl") plotdataImg, plotlayoutImg, imgWidth, imgHeight=loadImgPlot(imgInt) btnOpticalDisable=false else - traceImg=PlotlyBase.heatmap(x=[], y=[]) + traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImg=[traceImg] msgimg="" end @@ -643,7 +644,7 @@ include("./julia_imzML_visual.jl") plotdataImgT, plotlayoutImgT, imgWidth, imgHeight=loadImgPlot(imgIntT) btnOpticalDisable=false else - traceImg=PlotlyBase.heatmap(x=[], y=[]) + traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgT=[traceImg] msgtriq="" end @@ -670,7 +671,7 @@ include("./julia_imzML_visual.jl") plotdataImgT, plotlayoutImgT, imgWidth, imgHeight=loadImgPlot(imgIntT) btnOpticalDisable=false else - traceImg=PlotlyBase.heatmap(x=[], y=[]) + traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgT=[traceImg] msgtriq="" end @@ -699,7 +700,7 @@ include("./julia_imzML_visual.jl") plotdataImgComp, plotlayoutImgComp, _, _=loadImgPlot(imgIntComp) btnOpticalDisable=false else - traceImg=PlotlyBase.heatmap(x=[], y=[]) + traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgComp=[traceImg] msgimgComp="" end @@ -727,7 +728,7 @@ include("./julia_imzML_visual.jl") plotdataImgComp, plotlayoutImgComp, _, _=loadImgPlot(imgIntComp) btnOpticalDisable=false else - traceImg=PlotlyBase.heatmap(x=[], y=[]) + traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgComp=[traceImg] msgimgComp="" end @@ -755,7 +756,7 @@ include("./julia_imzML_visual.jl") plotdataImgTComp, plotlayoutImgTComp, _, _=loadImgPlot(imgIntTComp) btnOpticalDisable=false else - traceImg=PlotlyBase.heatmap(x=[], y=[]) + traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgTComp=[traceImg] msgtriqComp="" end @@ -783,7 +784,7 @@ include("./julia_imzML_visual.jl") plotdataImgTComp, plotlayoutImgTComp, _, _=loadImgPlot(imgIntTComp) btnOpticalDisable=false else - traceImg=PlotlyBase.heatmap(x=[], y=[]) + traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgTComp=[traceImg] msgtriqComp="" end diff --git a/config/env/global.jl b/config/env/global.jl index 0b24049..94f0b9d 100644 --- a/config/env/global.jl +++ b/config/env/global.jl @@ -1 +1,2 @@ -ENV["GENIE_ENV"] = "prod" +ENV["GENIE_ENV"] = "dev" +#ENV["GENIE_ENV"] = "prod" diff --git a/julia_imzML_visual.jl b/julia_imzML_visual.jl index 1316186..604fcb1 100644 --- a/julia_imzML_visual.jl +++ b/julia_imzML_visual.jl @@ -473,7 +473,7 @@ function meanSpectrumPlot(mzmlRoute::String) xSpectraMz=spectraMz[1,1] ySpectraMz=spectraMz[2,1] end - trace=PlotlyBase.scatter(x=xSpectraMz, y=ySpectraMz, mode="lines") + trace=PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz,marker=attr(size=1, color="blue", opacity=0.1)) plotdata=[trace] # We add the data from spectra to the plot plotlayout=layout return plotdata, plotlayout, xSpectraMz, ySpectraMz diff --git "a/ongoing_tasks.txt\n" "b/ongoing_tasks.txt\n" deleted file mode 100644 index 94a226e..0000000 --- "a/ongoing_tasks.txt\n" +++ /dev/null @@ -1,27 +0,0 @@ -PENDING - Rmsi & julia coherence with image creation and colorbar - colorbar revamp CURRENTLY ONGOING - Create function that makes the imzML from mzML files ? - Possibility to add multiple imzML to process at once ? - Add comparison image (rotate, transform, translate, transparency) CURRENTLY ONGOING - -DONE - Quicker start for julia GUI - Better UI - refined style of page - improved responsiveness - UI functionality added - Code cleanse - better code readability - Better blind search for images - 3d topology plots for images - 2d contour plots for images - Image flip upside down - Measure for the time it takes creating image or plot - Rotate vertical images - Easier input for files - TrIQ default values - Comparative for two views - Even faster initial boot and subsectuential boot - Multiple spectra plot types - Plot creation for spectra Per pixel of image diff --git a/scripts/build.jl b/scripts/build.jl new file mode 100644 index 0000000..86eafa9 --- /dev/null +++ b/scripts/build.jl @@ -0,0 +1,23 @@ +# scripts/build.jl +using PackageCompiler +using Pkg + +project_dir = dirname(@__DIR__) +Pkg.activate(project_dir) + +# Get all the direct dependencies from Project.toml +deps = keys(Pkg.project().dependencies) +sysimage_path = joinpath(project_dir, "build", "JuliaMSI_sysimage.so") +precompile_script = joinpath(project_dir, "scripts", "precompile.jl") + +@info "Starting system image compilation. This may take several minutes..." + +create_sysimage( + deps; + sysimage_path=sysimage_path, + precompile_execution_file=precompile_script +) + +@info "Compilation finished!" +@info "You can now run the fast-starting application using the following command:" +@info "julia --project=. -J build/JuliaMSI_sysimage.so start_MSI_GUI.jl" diff --git a/scripts/precompile.jl b/scripts/precompile.jl new file mode 100644 index 0000000..48b33da --- /dev/null +++ b/scripts/precompile.jl @@ -0,0 +1,45 @@ +# scripts/precompile.jl +using Pkg +Pkg.activate(dirname(@__DIR__)) + +using Genie +using HTTP + +@info "Precompiling application..." + +# Load the app. Mmap=false is recommended for compilation +Genie.loadapp(pwd(); Mmap=false) + +@info "Starting server for precompilation." +# Start the server in the background +server = up(1481, "127.0.0.1", async=true) + +# Give the server a moment to start up +sleep(20) + +base_url = "http://127.0.0.1:1481" +@info "Hitting routes to precompile..." + +try + # Make a GET request to the home page + response = HTTP.get(base_url * "/") + @info "GET / -> Status: $(response.status)" + + # === IMPORTANT === + # For best performance, you should add more HTTP requests here + # to hit ALL your important routes and API endpoints. + # This ensures the code for every page gets compiled. + # Example: + # HTTP.get(base_url * "/contact") + # HTTP.post(base_url * "/submit-data", [], "{\"key\":\"value\"}") + +catch e + @error "Could not hit routes during precompilation." exception=(e, catch_backtrace()) + +finally + # Stop the server + @info "Shutting down server." + down() +end + +@info "Precompilation tracing finished." diff --git a/src/Bitmap.jl b/src/Bitmap.jl new file mode 100644 index 0000000..a7edbf1 --- /dev/null +++ b/src/Bitmap.jl @@ -0,0 +1,399 @@ +# ******************************************************************** +# SaveBitmap +# ******************************************************************** +""" + SaveBitmap( name, pixMap, colorTable ) + +Save a discretized imzML slice as a bitmap file + +# Arguments +* ` name`: Full path name of target bitmap +* ` pixMap`: [UInt8] Bidimensional matrix with image info +* `colorTable`: [UInt32] Vector with RGB colors for each gray level + +# Examples +```julia +# Saves a bitmap with black and white alternate image squares +img = zeros(UInt8, 32, 32) +img[17:32,1:16] .= 1 +img[1:16,17:32] .= 1 +SaveBitmap( "test.bmp", img, UInt32[0, 0x00FFFFFF] ) +``` + +The following image will be created on your hard disk + +![](./Test.bmp) +""" +function SaveBitmap( name, + pixMap::Array{UInt8,2}, + colorTable::Array{UInt32,1} ) + + # Get image dimensions + dim = size( pixMap ) + if length( dim ) != 2 + return 0 + end + + # Compute row padding + padding = ( 4 - dim[1] & 0x3 ) & 0x3 + + # Compute file dimensions. Header = 14 + 40 + ( 256 * 4 ) = 1078 + offset = 1078 + imgBytes = dim[2] * ( dim[1] + padding ) + + # Create file + stream = open( name, "w" ) + + # Save file header + write( stream, UInt16( 0x4D42 ) ) + write( stream, UInt32[ offset + imgBytes, 0 , offset ] ) + + # Save info header + write( stream, UInt32[ 40, dim[1], dim[2], 0x80001, 0 ] ) + write( stream, UInt32[ imgBytes, 0, 0, 256, 0 ] ) + + # Save color table + write( stream, colorTable ) + if length( colorTable ) < 256 + fixTable = zeros( UInt32, 256 - length( colorTable ) ) + write( stream, fixTable ) + end + + # Save image pixels + if padding == 0 + for i = 1:dim[2] + write( stream, pixMap[:,i] ) + end + else + zeroPad = zeros( UInt8, padding ) + for i in 1:dim[2] + write( stream, pixMap[:,i] ) + write( stream, zeroPad ) + end + end + + # Close file + close( stream ) + +end + + +# ******************************************************************** +# FindMass +# massVector: mz Vector sorted in ascending order +# mass: target mz value +# tolerance: bi-axial tolerance to find mass +# ******************************************************************** +function FindMass( massVector, mass, tolerance ) + + index = Int( 0 ) + lower = Int( 1 ) + higher = Int( length( massVector ) ) + + while lower <= higher + + # Compute mid element + index = ( lower + higher ) ÷ 2 + + # Go to lower portion ? + if massVector[ index ] > ( mass + tolerance ) + higher = index - 1 + continue + end + + # Go to Higher portion? + if massVector[ index ] < ( mass - tolerance ) + lower = index + 1 + continue + end + + # mass was found + return index + end + + # mass was not found + return 0 +end + + +# ******************************************************************** +# GetSlice +# imzML: imzML data +# mass: mz target value +# tolerance: bi-axial tolerance to find mass +# ******************************************************************** +""" + GetSlice( imzML, mass, tolerance ) + +Extract a mz-image from an imzML image data array loaded with + `LoadImzml` function. The resulting image array preserves the same + data type of the y-axis vectors in the file. + +# Arguments +* ` imzML`: Image array loaded with `LoadImzml` function +* ` mass`: mz value for image extraction +* `tolerance`: Maximum lateral tolerance for a valid mz search + +# Examples +```julia +# Load DESI MSI Carcinoma 885.55 mz slice +spectra = LoadImzml( "80TopL, 50TopR, 70BottomL, 60BottomR-centroid.imzML" ) +slice = GetSlice( spectra, 885.55, 0.005 ) +``` +""" +function GetSlice( imzML, mass, tolerance ) + + # Alloc space for slice + width = maximum( imzML[1,:] ) + height = maximum( imzML[2,:] ) + image = zeros( Float64, width, height ) + + for i in 1:size( imzML )[2] + index = FindMass( imzML[3,i], mass, tolerance ) + if index != 0 + image[ imzML[1,i], imzML[2,i] ] = imzML[4,i][index] + end + end + + return image + +end + + +# ******************************************************************** +# IntQuant, Discretize image amplitude in 0:255 range +# slice: Image matriz in measured arbitrary units +# ******************************************************************** +""" + IntQuant( slice ) + +Zero Memory intensity quantizer. Discretize the image?s continuos + intensity range in discrete bins from 0 to 255 gray levels. + Returns a UInt8 bidimensional matrix + +# Arguments +* `slice`: slice returned by the `GetSlice` function + +# Examples +```julia +# Save DESI MSI Carcinoma 885.55 mz slice as a bitmap file +spectra = LoadImzml( "80TopL, 50TopR, 70BottomL, 60BottomR-centroid.imzML" ) +slice = GetSlice( spectra, 885.55, 0.005 +SaveBitmap( "Slice.bmp", IntQuant( slice ), ViridisPalette ) +``` + +The following image will be created on your hard disk + +![](./Slice.bmp) +""" +function IntQuant( slice ) + + # Compute scale factor for amplitude discretization + lower = minimum( slice ) + scale = 255 / maximum( slice ) + dim = size( slice ) + image = zeros( UInt8, dim[1], dim[2] ) + + for i in 1:length( slice ) + image[i] = convert( UInt8, floor( slice[i] * scale + 0.5 ) ) + end + + return image + +end + + + +# ******************************************************************** +# TrIQ, Discretize image amplitude in 0:255 range, grouping outliers +# in the highest bin +# slice: Image matrix in measured arbitrary units +# ******************************************************************** +function GetOutlierThres( slice, prob ) + + # Get bin width + low = minimum( slice ) + upp = maximum( slice ) + + # Compute histogram's bin count & separation + if ( upp - low + 1 ) >= 100 + bins = convert( Int32, 100 ) + step = ( upp + (upp-low)/(bins-1) - low ) / bins + else + bins = convert( Int32, ceil( upp ) - floor( low ) + 1 ) + step = 1 + end + + # Compute histogram + histCount = zeros( bins, 1 ) + for k in slice + index = convert( Int64, floor( ( k - low ) / step ) + 1 ) + histCount[ index ] += 1 + end + + # Get bin index, that accounts for desired probability + prob = 0.95 + delta = cumsum( histCount, dims=1 ) / sum( histCount ) .- prob + index = findmin( broadcast( abs, delta ) )[2][1] + + # Find max intensity image value + key = low + step * index + upp = -1 + + for k in slice[:] + if key > k && k > upp + upp = k + end + end + + return [ low, upp ] + +end + + +# ******************************************************************** +# Discretize image's intensity using a given intensity range +# slice: Image matrix in measured arbitrary units +# bounds: max and min intensity discretizing range +# depth: number of intensity discrete steps in final image +# ******************************************************************** +function SetPixelDepth( slice, bounds, depth ) + + # Compute intensity bins + bins = depth - 1 + limits = collect( + range( bounds[1], stop = bounds[2], length = depth ) ) + + # Reserve memory for output image + imgBytes = zeros( UInt8, size( slice ) ) + nPixels = length( slice ) + step = limits[2] - limits[1] + + # Set intensity depth + for k in 1:nPixels + + # Find bin + i::Int64 = floor( ( slice[k] - limits[1] ) / step ) + 1 + if i < bins + imgBytes[k] = i + ( slice[k] > limits[i] ) - 1 + else + imgBytes[k] = bins + end + + end + + return imgBytes +end + + +# ******************************************************************** +# Discretize image's intensity removing intensity outliers +# slice: Image matrix in measured arbitrary units +# depth: Number of intensity discrete steps in final image +# prob: Proportion of pixesl to take into account in discretization +# ******************************************************************** +""" + TrIQ( slice, depth, prob = 0.98 ) + +TrIQ intensity quantizer as described in _DOI 10.7717/peerj-cs.585_. + Discretize the image?s continuos intensity range in discrete bins + from 0 to 255 gray levels, groupping intensity outiers in the + highehst bin. Returns a UInt8 bidimensional matrix + +# Arguments +* `slice`: slice returned by the `GetSlice` function +* `depth`: Number ob discrete bins for intensity quantization +* ` prob`: Cummulative distribute function cutting value + +# Examples +```julia +# Save DESI MSI Carcinoma 885.55 mz slice as a bitmap file +spectra = LoadImzml( "80TopL, 50TopR, 70BottomL, 60BottomR-centroid.imzML" ) +slice = GetSlice( spectra, 885.55, 0.005 ) +SaveBitmap( "TrIQ.bmp", TrIQ( slice, 256, 0.95 ), ViridisPalette ) +``` + +The following image will be created on your hard disk + +![](./TrIQ.bmp) +""" +function TrIQ( slice, depth, prob ) + + return SetPixelDepth( + slice, + GetOutlierThres( slice, prob ), + depth + ) + +end + + +# ******************************************************************** +# Viridis color palette 256 color levels +# ******************************************************************** +ViridisPalette = UInt32[ + 0x440154, 0x440256, 0x450457, 0x450559, + 0x46075A, 0x46085C, 0x460A5D, 0x460B5E, + 0x470D60, 0x470E61, 0x471063, 0x471164, + 0x471365, 0x481467, 0x481668, 0x481769, + 0x48186A, 0x481A6C, 0x481B6D, 0x481C6E, + 0x481D6F, 0x481F70, 0x482071, 0x482173, + 0x482374, 0x482475, 0x482576, 0x482677, + 0x482878, 0x482979, 0x472A7A, 0x472C7A, + 0x472D7B, 0x472E7C, 0x472F7D, 0x46307E, + 0x46327E, 0x46337F, 0x463480, 0x453581, + 0x453781, 0x453882, 0x443983, 0x443A83, + 0x443B84, 0x433D84, 0x433E85, 0x423F85, + 0x424086, 0x424186, 0x414287, 0x414487, + 0x404588, 0x404688, 0x3F4788, 0x3F4889, + 0x3E4989, 0x3E4A89, 0x3E4C8A, 0x3D4D8A, + 0x3D4E8A, 0x3C4F8A, 0x3C508B, 0x3B518B, + 0x3B528B, 0x3A538B, 0x3A548C, 0x39558C, + 0x39568C, 0x38588C, 0x38598C, 0x375A8C, + 0x375B8D, 0x365C8D, 0x365D8D, 0x355E8D, + 0x355F8D, 0x34608D, 0x34618D, 0x33628D, + 0x33638D, 0x32648E, 0x32658E, 0x31668E, + 0x31678E, 0x31688E, 0x30698E, 0x306A8E, + 0x2F6B8E, 0x2F6C8E, 0x2E6D8E, 0x2E6E8E, + 0x2E6F8E, 0x2D708E, 0x2D718E, 0x2C718E, + 0x2C728E, 0x2C738E, 0x2B748E, 0x2B758E, + 0x2A768E, 0x2A778E, 0x2A788E, 0x29798E, + 0x297A8E, 0x297B8E, 0x287C8E, 0x287D8E, + 0x277E8E, 0x277F8E, 0x27808E, 0x26818E, + 0x26828E, 0x26828E, 0x25838E, 0x25848E, + 0x25858E, 0x24868E, 0x24878E, 0x23888E, + 0x23898E, 0x238A8D, 0x228B8D, 0x228C8D, + 0x228D8D, 0x218E8D, 0x218F8D, 0x21908D, + 0x21918C, 0x20928C, 0x20928C, 0x20938C, + 0x1F948C, 0x1F958B, 0x1F968B, 0x1F978B, + 0x1F988B, 0x1F998A, 0x1F9A8A, 0x1E9B8A, + 0x1E9C89, 0x1E9D89, 0x1F9E89, 0x1F9F88, + 0x1FA088, 0x1FA188, 0x1FA187, 0x1FA287, + 0x20A386, 0x20A486, 0x21A585, 0x21A685, + 0x22A785, 0x22A884, 0x23A983, 0x24AA83, + 0x25AB82, 0x25AC82, 0x26AD81, 0x27AD81, + 0x28AE80, 0x29AF7F, 0x2AB07F, 0x2CB17E, + 0x2DB27D, 0x2EB37C, 0x2FB47C, 0x31B57B, + 0x32B67A, 0x34B679, 0x35B779, 0x37B878, + 0x38B977, 0x3ABA76, 0x3BBB75, 0x3DBC74, + 0x3FBC73, 0x40BD72, 0x42BE71, 0x44BF70, + 0x46C06F, 0x48C16E, 0x4AC16D, 0x4CC26C, + 0x4EC36B, 0x50C46A, 0x52C569, 0x54C568, + 0x56C667, 0x58C765, 0x5AC864, 0x5CC863, + 0x5EC962, 0x60CA60, 0x63CB5F, 0x65CB5E, + 0x67CC5C, 0x69CD5B, 0x6CCD5A, 0x6ECE58, + 0x70CF57, 0x73D056, 0x75D054, 0x77D153, + 0x7AD151, 0x7CD250, 0x7FD34E, 0x81D34D, + 0x84D44B, 0x86D549, 0x89D548, 0x8BD646, + 0x8ED645, 0x90D743, 0x93D741, 0x95D840, + 0x98D83E, 0x9BD93C, 0x9DD93B, 0xA0DA39, + 0xA2DA37, 0xA5DB36, 0xA8DB34, 0xAADC32, + 0xADDC30, 0xB0DD2F, 0xB2DD2D, 0xB5DE2B, + 0xB8DE29, 0xBADE28, 0xBDDF26, 0xC0DF25, + 0xC2DF23, 0xC5E021, 0xC8E020, 0xCAE11F, + 0xCDE11D, 0xD0E11C, 0xD2E21B, 0xD5E21A, + 0xD8E219, 0xDAE319, 0xDDE318, 0xDFE318, + 0xE2E418, 0xE5E419, 0xE7E419, 0xEAE51A, + 0xECE51B, 0xEFE51C, 0xF1E51D, 0xF4E61E, + 0xF6E620, 0xF8E621, 0xFBE723, 0xFDE725 ] + diff --git a/src/Common.jl b/src/Common.jl new file mode 100644 index 0000000..aa26b94 --- /dev/null +++ b/src/Common.jl @@ -0,0 +1,94 @@ +# ******************************************************************* +# VectorConfig, structure with axis decoding instructions +# ******************************************************************* +mutable struct SpecDim + Format::DataType # Float32, Float64 + Packed::Bool # 0: none 1: zlib compression + Axis::Int32 # 1: m/z 2: Amplitude + Skip::Int64 # Bytes to skip +end + +# ******************************************************************* +# Fill VectorConfig fields from "cvParam" tags +# ******************************************************************* +function ConfigureSpecDim( stream ) + + # Vector configuration + axis = SpecDim( Float64, false, 1, 0 ) + + # Initial values + offset = position( stream ) + currLine = "" + matchInfo = RegexMatch + + while true + + # Next field + currLine = readline( stream ) + matchInfo = match( r"^\s*<(cvParam)", currLine ) + + if matchInfo === nothing + matchInfo = match( r"^\s*", currLine ) + axis.Skip = position( stream ) - offset - + length( currLine ) + length( matchInfo.match ) + return axis + end + + index = length( matchInfo.captures[1] ) + matchInfo = GetAttribute( currLine[index:end], "accession" ) + + if matchInfo.captures[1] == "MS:1000515" # intensity array + axis.Axis = 2 + continue + + elseif matchInfo.captures[1] == "MS:1000519" # 32-bit integer + axis.Format = Int32 + continue + + elseif matchInfo.captures[1] == "MS:1000521" # 32-bit float + axis.Format = Float32 + continue + + elseif matchInfo.captures[1] == "MS:1000522" # 64-bit integer + axis.Format = Int64 + continue + + elseif matchInfo.captures[1] == "MS:1000574" # zlib compresion + axis.Packed = true + continue + + end + end +end + + +# ******************************************************************* +# Read lines in file until regex match +# stream: Source file stream +# regex: target tag e.g. "binaryDataArray" +# ******************************************************************* +function FindTag( stream, regex ) + + while true + isTag = match( regex, readline( stream ) ) + if isTag !== nothing + return isTag + end + end +end + + +# ******************************************************************* +# Retrieve attribute/value pair +# source: String with atr="value" pairs +# tag: Optional attribute, e.g. "\\spectrum" +# ******************************************************************* +function GetAttribute( source, tag = "([^=]+)" ) + + # Build regex and retrieve attribute/value pair + regStr = Regex( "\\s" * tag * "=\"([^\"]*)\"" ) + return( match( regStr, source ) ) + +end + + diff --git a/src/DataAccess.jl b/src/DataAccess.jl new file mode 100644 index 0000000..5977248 --- /dev/null +++ b/src/DataAccess.jl @@ -0,0 +1,82 @@ +# src/DataAccess.jl + +""" + load_spectra(fileName::String) + +Eagerly loads all spectra from a .mzML or .imzML file into a 2xN matrix. +This function follows the eager-loading pattern of the original `LoadMzml` +for backward compatibility and for analyses that require all data in memory. + +Each column in the returned matrix represents a single spectrum: +- Row 1: m/z array +- Row 2: Intensity array + +For .imzML files, the (x,y) spatial coordinates are discarded. + +# Arguments +* `fileName`: Full path to the .mzML or .imzML file. + +# Returns +- An `Array{Any, 2}` containing all spectra. +""" +function load_spectra(fileName::String) + if endswith(lowercase(fileName), ".mzml") + # Detected .mzML file. Using the existing eager loader. + return LoadMzml(fileName) + + elseif endswith(lowercase(fileName), ".imzml") + # Detected .imzML file. Eagerly loading all spectra from .ibd. + + # Lazily load metadata to get offsets and counts + imzml_data = load_imzml(fileName) + + try + num_spectra = length(imzml_data.spectra_metadata) + if num_spectra == 0 + return Array{Any}(undef, (2, 0)) + end + + spectra_matrix = Array{Any}(undef, (2, num_spectra)) + + hIbd = imzml_data.ibd_handle + mz_format = imzml_data.mz_format + int_format = imzml_data.intensity_format + + # For performance, pre-allocate one large buffer for reading. + # This avoids re-allocating memory for each spectrum inside the loop. + max_points = maximum(meta.mz_count for meta in imzml_data.spectra_metadata) + mz_array_buffer = Array{mz_format}(undef, max_points) + intensity_array_buffer = Array{int_format}(undef, max_points) + + for i in 1:num_spectra + meta = imzml_data.spectra_metadata[i] + + # Create a view into the buffer with the correct size for this spectrum + current_mz_array = view(mz_array_buffer, 1:meta.mz_count) + current_int_array = view(intensity_array_buffer, 1:meta.intensity_count) + + # Read binary data directly into the sized views + seek(hIbd, meta.mz_offset) + read!(hIbd, current_mz_array) + + seek(hIbd, meta.intensity_offset) + read!(hIbd, current_int_array) + + # Store a copy in the final matrix. A copy is necessary because + # the buffer is overwritten in the next iteration. + spectra_matrix[1, i] = copy(current_mz_array) + spectra_matrix[2, i] = copy(current_int_array) + end + + return spectra_matrix + finally + # Ensure the .ibd file handle is closed, as this is an eager load. + if isopen(imzml_data.ibd_handle) + close(imzml_data.ibd_handle) + end + end + + else + error("Unsupported file type. Please provide a .mzML or .imzML file.") + end +end diff --git a/src/MzmlConverter.jl b/src/MzmlConverter.jl new file mode 100644 index 0000000..16a39ba --- /dev/null +++ b/src/MzmlConverter.jl @@ -0,0 +1,270 @@ +# src/MzmlConverter.jl + +""" +This file contains the workflow for converting .mzML files (with one spectrum per pixel) +into a proper .imzML/.ibd file pair, using a separate synchronization file. +It replicates the functionality of the project's original R scripts that use MALDIquant. +""" + +using CSV, DataFrames + +# Note: This file assumes that `LoadSpectra` from DataAccess.jl and the necessary +# structs/functions from ParserHelpers.jl, mzML.jl, and imzML.jl are available +# in the execution context (e.g., included in the main app.jl). + +# A struct to hold the processed pixel data before exporting +struct ProcessedPixel + coords::Tuple{Int, Int} + mz::Vector{Float64} # Assuming m/z is consistent, can be optimized later + intensity::Vector{Float32} +end + +""" + GetMzmlScanTime(fileName::String) + +Parses a .mzML file to extract the scan start time for each spectrum. +This is a Julia implementation of the `GetMzmlScanTime` function from the R scripts. + +# Arguments +* `fileName`: Path to the .mzML file. + +# Returns +- A `Matrix{Int64}` where each row is `[spectrum_index, time_in_milliseconds]`. +""" +function GetMzmlScanTime(fileName::String) + spec_count = 0 + times = Tuple{Int64, Int64}[] + + open(fileName, "r") do stream + file_content = read(stream, String) + + # First, find the total number of spectra to pre-allocate + spectrum_list_match = match(r" tag or count attribute.") + end + total_spectra = parse(Int, spectrum_list_match.captures[1]) + sizehint!(times, total_spectra) + + # Iterate over each spectrum block + for spectrum_match in eachmatch(r"]*>.*?<\/spectrum>", file_content, overlay=true) + spec_block = spectrum_match.match + # The R script appears to treat the 0-based mzML index as 1-based, so we add 1. + spec_index = parse(Int, spectrum_match.captures[1]) + 1 + + # Find the scan start time within the spectrum block + time_match = match(r" x[1]) # Ensure scans are sorted by index + first_time = times[1][2] + for i in eachindex(times) + times[i] = (times[i][1], times[i][2] - first_time) + end + end + end # open + + # Convert vector of tuples to a matrix + return permutedims(hcat(collect.(times)...)) +end + +""" + MatchAcquireTime(sync_file_path::String, scans::Matrix{Int64}) + +Correlates pixel acquisition times from a synchronization file with scan +acquisition times from an mzML file. + +This is a Julia implementation of the `MatchAcquireTime` function from the R scripts. + +# Arguments +* `sync_file_path`: Path to the synchronization file (.txt). +* `scans`: A matrix of scan times, as returned by `GetMzmlScanTime`. + +# Returns +- A `Matrix{Int64}` where each row is `[pixel_index, pixel_time_ms, first_scan_index, last_scan_index]`. +""" +function MatchAcquireTime(sync_file_path::String, scans::Matrix{Int64}) + if !isfile(sync_file_path) + error("Synchronization file not found: $sync_file_path") + end + + # Load xyz stage pixel delays from the sync file, skipping the first 2 header lines. + pixel_df = CSV.read(sync_file_path, DataFrame, header=false, skipto=3) + pixel = Matrix(pixel_df) + + # Normalize pixel times so the first is at t=0 + pixel[:, 2] .-= pixel[1, 2] + + num_pixels = size(pixel, 1) + num_scans = size(scans, 1) + + index_matrix = zeros(Int64, num_pixels, 2) + + scan_row = 1 + for pixel_row in 1:(num_pixels - 1) + start_scan = scan_row + + # Find the scan that ends after the *next* pixel starts + while scan_row <= num_scans && scans[scan_row, 2] < pixel[pixel_row + 1, 2] + scan_row += 1 + end + + index_matrix[pixel_row, 1] = start_scan + index_matrix[pixel_row, 2] = scan_row - 1 + + # The next pixel's scans start from the one that crossed the boundary + scan_row = max(1, scan_row - 1) + end + + # Assign scans for the last pixel + index_matrix[num_pixels, 1] = scan_row + index_matrix[num_pixels, 2] = num_scans + + # Handle cases where some pixels have no scans by creating an empty range + for i in 1:num_pixels + if index_matrix[i, 2] < index_matrix[i, 1] + index_matrix[i, 2] = index_matrix[i, 1] - 1 + end + end + + return hcat(pixel, index_matrix) +end + + +function RenderPixel(pixel_info, scans, spectra, scan_time_deltas, pixel_time_deltas) + first_scan = pixel_info[3] + last_scan = pixel_info[4] + num_actions = last_scan - first_scan + + # Use the m/z array of the first scan as the reference + mz_array = spectra[1, first_scan] + new_intensity = zeros(Float32, length(mz_array)) + + if num_actions == 0 # Single scan contributes to the pixel + scale = pixel_time_deltas[pixel_info[1]] / scan_time_deltas[first_scan] + new_intensity .+= spectra[2, first_scan] .* scale + + elseif num_actions == 1 # Two partial scans contribute + # First partial scan + scale1 = (scans[last_scan, 2] - pixel_info[2]) / scan_time_deltas[first_scan] + new_intensity .+= spectra[2, first_scan] .* scale1 + + # Second partial scan + next_pixel_time = pixel_info[2] + pixel_time_deltas[pixel_info[1]] + scale2 = (next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan] + new_intensity .+= spectra[2, last_scan] .* scale2 + + elseif num_actions > 1 # Multiple scans contribute + # First partial scan + scale1 = (scans[first_scan + 1, 2] - pixel_info[2]) / scan_time_deltas[first_scan] + new_intensity .+= spectra[2, first_scan] .* scale1 + + # Full scans in the middle + for i in (first_scan + 1):(last_scan - 1) + new_intensity .+= spectra[2, i] + end + + # Last partial scan + next_pixel_time = pixel_info[2] + pixel_time_deltas[pixel_info[1]] + scale2 = (next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan] + new_intensity .+= spectra[2, last_scan] .* scale2 + end + + return (mz_array, new_intensity) +end + +function ConvertMzmlToImzml(source_file::String, timing_matrix::Matrix{Int64}, scans::Matrix{Int64}) + # Get image width from timing info + width = findfirst(i -> timing_matrix[i, 1] - timing_matrix[i-1, 1] != 1, 2:size(timing_matrix, 1)) + height = size(timing_matrix, 1) ÷ width + + # Load the full mzML data + spectra = LoadSpectra(source_file) + + # Pre-calculate time deltas + scan_time_deltas = diff(scans[:, 2]) + pixel_time_deltas = diff(timing_matrix[:, 2]) + # Append a final delta for the last element + push!(scan_time_deltas, scan_time_deltas[end]) + push!(pixel_time_deltas, pixel_time_deltas[end]) + + processed_pixels = ProcessedPixel[] + sizehint!(processed_pixels, size(timing_matrix, 1)) + + for i in 1:size(timing_matrix, 1) + pixel_info = timing_matrix[i, :] + first_scan = pixel_info[3] + last_scan = pixel_info[4] + + # Skip if there are no scans for this pixel + (first_scan > last_scan) && continue + + # Calculate coordinates + x = ((pixel_info[1] - 1) % width) + 1 + y = fld(pixel_info[1] - 1, width) + 1 + + mz, intensity = RenderPixel(pixel_info, scans, spectra, scan_time_deltas, pixel_time_deltas) + push!(processed_pixels, ProcessedPixel((x, y), mz, intensity)) + end + + return processed_pixels, (width, height) +end + +function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims::Tuple{Int, Int}) + @warn "ExportImzml is not yet implemented. The processed pixel data has been generated but not saved to .imzML/.ibd files." + # TODO: Implement the logic to write the .imzML (XML) and .ibd (binary) files. + # 1. Open target_file.imzML and target_file.ibd for writing. + # 2. Write all m/z and intensity arrays sequentially to the .ibd file, tracking offsets and lengths. + # 3. Write the .imzML XML structure, including: + # - Boilerplate headers + # - , , + # - A with a for each pixel. + # - Each must contain cvParams for x/y coords and external data pointers to the .ibd file. + return false +end + +""" + ImportMzmlFile(source_file::String, sync_file::String, target_file::String) + +Main workflow function to convert a .mzML file to an .imzML file. + +# Arguments +* `source_file`: Path to the input .mzML file. +* `sync_file`: Path to the synchronization text file. +* `target_file`: Path for the output .imzML file (the .ibd will be named accordingly). +""" +function ImportMzmlFile(source_file::String, sync_file::String, target_file::String) + println("Step 1: Getting scan times from .mzML file...") + scans = GetMzmlScanTime(source_file) + + println("Step 2: Matching acquisition times...") + timing_matrix = MatchAcquireTime(sync_file, scans) + + println("Step 3: Converting spectra...") + processed_pixels, (width, height) = ConvertMzmlToImzml(source_file, timing_matrix, scans) + + # Flip image vertically to match R script output + for i in eachindex(processed_pixels) + x, y = processed_pixels[i].coords + processed_pixels[i] = ProcessedPixel((x, height - y + 1), processed_pixels[i].mz, processed_pixels[i].intensity) + end + + println("Step 4: Exporting to .imzML/.ibd format...") + success = ExportImzml(target_file, processed_pixels, (width, height)) + + if success + println("Conversion successful: $target_file") + else + println("Conversion failed. Exporting is not yet implemented.") + end + return success +end diff --git a/src/ParserHelpers.jl b/src/ParserHelpers.jl new file mode 100644 index 0000000..af53599 --- /dev/null +++ b/src/ParserHelpers.jl @@ -0,0 +1,122 @@ +# src/ParserHelpers.jl + +""" +This file provides common helper functions for parsing mzML and imzML files. +""" + +# ============================================================================ +# +# Data Structures and Low-Level Parser Helpers +# +# ============================================================================ + +""" + SpecDim + +A struct to hold configuration for a spectral data axis (e.g., m/z or intensity). +""" +mutable struct SpecDim + Format::Type + Packed::Bool + Axis::Int + Skip::Int +end + +""" + find_tag(stream, regex::Regex) + +Reads a stream line-by-line until a line matches the provided regex. + +# Returns +- A `RegexMatch` object if a match is found, otherwise throws an error. +""" +function find_tag(stream, regex::Regex) + while !eof(stream) + line = readline(stream) + isTag = match(regex, line) + if isTag !== nothing + return isTag + end + end + error("Tag not found for regex: $regex") +end + +""" + get_attribute(source::String, tag::String = "([^=]+)") + +Retrieves an attribute's value from an XML tag string. + +# Returns +- A `RegexMatch` object containing the attribute and its value. +""" +function get_attribute(source::AbstractString, tag::String = "([^=]+)") + # Construct the regex pattern string + pattern_str = "\s" * tag * "=\"([^"]*)\"" + regStr = Regex(pattern_str) + return match(regStr, source) +end + +""" + configure_spec_dim(stream) + +Fills a `SpecDim` struct by parsing `cvParam` tags from the stream. +""" +function configure_spec_dim(stream) + axis = SpecDim(Float64, false, 1, 0) + offset = position(stream) + + while !eof(stream) + currLine = readline(stream) + matchInfo = match(r"^\s*<(cvParam)", currLine) + + if matchInfo === nothing + matchInfo = match(r"^\s*", currLine) + axis.Skip = position(stream) - offset - length(currLine) + length(matchInfo.match) + return axis + end + + index = length(matchInfo.captures[1]) + attr_match = get_attribute(currLine[index:end], "accession") + + if attr_match !== nothing + accession = attr_match.captures[1] + if accession == "MS:1000515" # intensity array + axis.Axis = 2 + elseif accession == "MS:1000519" # 32-bit integer + axis.Format = Int32 + elseif accession == "MS:1000521" # 32-bit float + axis.Format = Float32 + elseif accession == "MS:1000522" # 64-bit integer + axis.Format = Int64 + elseif accession == "MS:1000574" # zlib compression + axis.Packed = true + end + end + end + return axis # Should be unreachable if file is well-formed +end + +# ============================================================================ +# +# Data Structures and Helpers for mzML lazy parsing +# +# ============================================================================ + +mutable struct CVParams + format::Type + is_compressed::Bool + axis_type::Symbol +end + +function update_cv_params!(params::CVParams, acc::String) + if acc == "MS:1000514"; params.axis_type = :mz; + elseif acc == "MS:1000515"; params.axis_type = :intensity; + elseif acc == "MS:1000519"; params.format = Int32; + elseif acc == "MS:1000521"; params.format = Float32; + elseif acc == "MS:1000522"; params.format = Int64; + elseif acc == "MS:1000523"; params.format = Float64; + elseif acc == "MS:1000574"; params.is_compressed = true; + end +end + + diff --git a/src/imzML.jl b/src/imzML.jl new file mode 100644 index 0000000..2c87316 --- /dev/null +++ b/src/imzML.jl @@ -0,0 +1,699 @@ + +using Images, ImageFiltering, StatsBase, Statistics, CairoMakie, ColorSchemes, DataFrames, CSV, Printf, Dates +include("ParserHelpers.jl") + +# --- Lazy Loading Data Structures --- + +# Struct to hold metadata for a single spectrum, for lazy loading. +struct SpectrumMetadata + x::Int32 + y::Int32 + mz_offset::Int64 + intensity_offset::Int64 + mz_count::Int32 + intensity_count::Int32 +end + +# Struct to hold the parsed imzML metadata and the handle to the .ibd file. +mutable struct ImzMLData + ibd_handle::IO + mz_format::Type + intensity_format::Type + spectra_metadata::Vector{SpectrumMetadata} + width::Int + height::Int + mz_is_first::Bool # To know the order in the ibd file + + function ImzMLData(ibd_handle::IO, mz_format::Type, intensity_format::Type, spectra_metadata::Vector{SpectrumMetadata}, width::Int, height::Int, mz_is_first::Bool) + obj = new(ibd_handle, mz_format, intensity_format, spectra_metadata, width, height, mz_is_first) + finalizer(obj) do o + if isopen(o.ibd_handle) + close(o.ibd_handle) + end + end + return obj + end +end + + +# --- Extracted from imzML.jl --- + +""" +This file provides a library for parsing `.imzML` and `.ibd` files in pure Julia. +It is intended to be included by a parent script. + +Core Functions: +- `load_imzml`: The main function that orchestrates the parsing. +- Helper functions for reading XML metadata and binary spectral data. +""" + + +# ============================================================================ +# +# +# imzML Parser Implementation +# +# ============================================================================ + +""" + axes_config_img(stream) + +Determines the storage order of the m/z and intensity arrays. +""" +function axes_config_img(stream) + tag = find_tag(stream, r"^\s*<(referenceableParamGroup )") + value = get_attribute(tag.captures[1], "intensityArray") + order = 1 + (value !== nothing) + + axis = Array{SpecDim,1}(undef, 2) + axis[order] = configure_spec_dim(stream) + + find_tag(stream, r"^\s*<(referenceableParamGroup )") + axis[xor(order, 3)] = configure_spec_dim(stream) + return axis +end + +""" + get_img_dimensions(stream) + +Reads the maximum X and Y dimensions and total spectrum count from the metadata. +""" +function get_img_dimensions(stream) + find_tag(stream, r"^\s*<(scanSettings )") + n = 2 + dim = [0, 0, 0] + + while n > 0 && !eof(stream) + currLine = readline(stream) + if occursin("` tag, ignoring attribute values. +""" +function get_spectrum_tag_offset(stream) + offset = position(stream) + tag = find_tag(stream, r"^\s*") + skip[8] = position(stream) - offset + return skip +end + + +""" + load_imzml(file_path::String) +Main function to parse an `.imzML`/.ibd file pair and prepare for lazy loading. +It parses the metadata from the `.imzML` file, including spectrum coordinates and +byte offsets for data in the `.ibd` file, but does not load the spectral data itself. + +# Returns +- An `ImzMLData` struct containing the open `.ibd` file handle and all necessary metadata + for on-demand data loading. +""" +function load_imzml(file_path::String) + if !isfile(file_path) + error("Provided path is not a file: $(file_path)") + end + + ibd_path = replace(file_path, ".imzML" => ".ibd") + if !isfile(ibd_path) + error("Corresponding .ibd file not found for: $(file_path)") + end + + stream = open(file_path) + hIbd = open(ibd_path) + + # We don't use a try/finally block because we need to return the open hIbd handle. + # The finalizer on the ImzMLData struct will be responsible for closing it. + + axis = axes_config_img(stream) + imgDim = get_img_dimensions(stream) + width, height, num_spectra = imgDim + + # Determine which format is for m/z and which for intensity from the referenceableParamGroups + mz_config_idx = findfirst(a -> a.Axis == 1, axis) + int_config_idx = findfirst(a -> a.Axis == 2, axis) + mz_format = axis[mz_config_idx].Format + intensity_format = axis[int_config_idx].Format + + # The original parsing logic is highly optimized and relies on fixed offsets within the XML structure. + # We reuse it to get the initial .ibd file offset and the structural attributes of the tags. + start_of_spectra_xml = position(stream) + attr = get_spectrum_attributes(stream, hIbd) + + # get_spectrum_attributes moves the hIbd pointer to the start of the first spectrum's binary data. + current_ibd_offset = position(hIbd) + + # We need to re-scan the spectra list from the beginning to get metadata for each spectrum. + seek(stream, start_of_spectra_xml) + + spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra) + + # attr[3] is 4 if the first binary data array is intensity, 3 if it's m/z. + mz_is_first = attr[3] == 3 + + for k in 1:num_spectra + # These skips are based on the structure of the first spectrum tag, assuming all are identical. + skip(stream, attr[5]) # Skip to X coordinate value + val_tag = find_tag(stream, r"value=\"(\d+)\"") + x = parse(Int32, val_tag.captures[1]) + + skip(stream, attr[6]) # Skip to Y coordinate value + val_tag = find_tag(stream, r"value=\"(\d+)\"") + y = parse(Int32, val_tag.captures[1]) + + skip(stream, attr[7]) # Skip to array length value + val_tag = find_tag(stream, r"value=\"(\d+)\"") + nPoints = parse(Int32, val_tag.captures[1]) + + mz_len_bytes = nPoints * sizeof(mz_format) + int_len_bytes = nPoints * sizeof(intensity_format) + + local mz_offset, int_offset + if mz_is_first + mz_offset = current_ibd_offset + int_offset = mz_offset + mz_len_bytes + else + int_offset = current_ibd_offset + mz_offset = int_offset + int_len_bytes + end + + spectra_metadata[k] = SpectrumMetadata(x, y, mz_offset, int_offset, nPoints, nPoints) + + # Advance the offset for the next spectrum's data. + current_ibd_offset += mz_len_bytes + int_len_bytes + + skip(stream, attr[8]) # Skip to the end of the spectrum tag + end + + # Close the .imzML file stream, but leave the .ibd stream open for lazy reading. + close(stream) + + return ImzMLData(hIbd, mz_format, intensity_format, spectra_metadata, width, height, mz_is_first) +end + +# --- End of content from imzML.jl --- + + +# --- Start of content from Imaging_Normalization.jl --- + +# ============================================================================= +# +# Image Slice Extraction +# +# ============================================================================= + +""" + find_mass(mz_array, intensity_array, target_mass, tolerance) + +Finds the intensity of the most intense peak within a mass tolerance window. +This modernized version is more robust than a simple binary search as it +correctly handles multiple peaks within the tolerance window. + +# Returns +- The intensity (`Float64`) of the peak if found, otherwise `0.0`. +""" +function find_mass(mz_array, intensity_array, target_mass, tolerance) + lower_bound = target_mass - tolerance + upper_bound = target_mass + tolerance + + max_intensity = 0.0 + found = false + + # Iterate through the spectrum to find the highest intensity peak in the window + for i in eachindex(mz_array) + if lower_bound <= mz_array[i] <= upper_bound + if intensity_array[i] > max_intensity + max_intensity = intensity_array[i] + found = true + end + end + end + + return found ? max_intensity : 0.0 +end + +""" + load_slices(folder, masses, tolerance) + +Loads image slices for multiple masses from all `.imzML` files in a directory. +This function is optimized to read the binary data file (`.ibd`) only once per file. +It iterates through each spectrum, reads its data, and finds all target masses +before proceeding to the next spectrum, minimizing disk I/O. +""" +function load_slices(folder, masses, tolerance) + files = filter(f -> endswith(f, ".imzML"), readdir(folder, join=true)) + if isempty(files) + @warn "No .imzML files found in the specified directory: $folder" + return (Array{Any}(undef, 0, 0), String[]) + end + n_files = length(files) + n_slices = length(masses) + + img_list = Array{Any}(undef, n_files, n_slices) + names = String[] + + for (i, file) in enumerate(files) + name = basename(file) + push!(names, name) + @info "Processing $(i)/$(n_files): $(name)" + + imzML_data = @time load_imzml(file) # This is fast, it's fine. + + # Create empty images for all slices for the current file + current_file_slices = [zeros(Float64, imzML_data.width, imzML_data.height) for _ in 1:n_slices] + + hIbd = imzML_data.ibd_handle + mz_format = imzML_data.mz_format + int_format = imzML_data.intensity_format + + # Pre-allocate arrays to be reused for each spectrum + mz_array = Array{mz_format}(undef, 0) + intensity_array = Array{int_format}(undef, 0) + + # Iterate through each spectrum ONCE + for meta in imzML_data.spectra_metadata + # Resize arrays if the number of points in the spectrum has changed + if length(mz_array) != meta.mz_count + resize!(mz_array, meta.mz_count) + resize!(intensity_array, meta.intensity_count) + end + + # Read spectrum data ONCE from the .ibd file + seek(hIbd, meta.mz_offset) + read!(hIbd, mz_array) + seek(hIbd, meta.intensity_offset) + read!(hIbd, intensity_array) + + # Now, check for all masses of interest in this single spectrum + for (j, mass) in enumerate(masses) + intensity = find_mass(mz_array, intensity_array, mass, tolerance) + if intensity > 0.0 + if 1 <= meta.x <= imzML_data.width && 1 <= meta.y <= imzML_data.height + current_file_slices[j][meta.x, meta.y] = intensity + end + end + end + end # end of spectra loop + + # Assign the generated images to the main list + for j in 1:n_slices + img_list[i, j] = current_file_slices[j] + end + + end # end of files loop + + return (img_list, names) +end + +# ============================================================================ +# +# +# Image Processing and Normalization +# +# ============================================================================ + +function get_outlier_thres(img, prob=0.98) + # DO NOT filter zeros. Use all pixel values like R does. + int_values = vec(img) + low = minimum(int_values) + upp = maximum(int_values) + + # Create histogram bins + if upp - low + 1 >= 100 + bins = 100 + brk = range(low, stop=upp + (upp - low)/(bins - 1), length=bins + 1) + else + brk = collect(floor(low):(ceil(upp) + 1)) + end + + # Compute histogram with RIGHT-CLOSED = FALSE to mimic R's right=FALSE + # This makes intervals [a, b) instead of the default [a, b] + h = fit(Histogram, int_values, brk, closed=:left) # Key change: closed=:left + + # Check if histogram is valid + if isempty(h.weights) || sum(h.weights) == 0 + return (low, upp) + end + + # Replicate R's algorithm exactly + cum_counts = cumsum(h.weights) / sum(h.weights) + # Calculate the difference from the target probability + top = prob .- cum_counts + delta = abs.(top) + # Find the index where the difference is minimized + min_delta_index = findfirst(x -> x == minimum(delta), delta) + + # R adds 1 to this index: index <- 1 + which(...)[1] + target_bin_index = min_delta_index + 1 + # Get the upper edge of the target bin + target_bin_upper_edge = h.edges[1][target_bin_index] + + # Find the maximum data value that is strictly less than this edge + # This mimics: max( intMap[ intMap < h$breaks[index] ] ) + values_below_edge = filter(x -> x < target_bin_upper_edge, int_values) + actual_threshold = isempty(values_below_edge) ? low : maximum(values_below_edge) + + return (low, actual_threshold) +end + +function set_pixel_depth(img, bounds, depth) + min_val, max_val = bounds + bins = depth - 1 + + if min_val >= max_val + return zeros(UInt8, size(img)) + end + + # Create intensity bins + range_vals = range(min_val, stop=max_val, length=depth)[2:depth] + + # Assign each pixel to a bin + result = similar(img, UInt8) # Use similar to create an array of the same type and size + for i in eachindex(img) + if img[i] <= min_val + result[i] = 0 + else + bin_idx = findfirst(x -> img[i] <= x, range_vals) + result[i] = bin_idx === nothing ? bins : bin_idx - 1 + end + end + + return result +end + +""" + norm_slices_hist(slices, bins; prob=0.98) + +Normalizes a set of image slices based on a shared histogram range. +""" +function norm_slices_hist(slices, bins; prob=0.98) + n_files, n_masses = size(slices) + norm_img = similar(slices) + mass_bounds = [] # This will store bounds for EACH mass + + # Calculate bounds for each mass across all files + for mass_idx in 1:n_masses + # Get all slices for this specific mass across all files + mass_slices = [slices[i, mass_idx] for i in 1:n_files] + all_vals = reduce(vcat, [vec(s) for s in mass_slices]) + + # Calculate global bounds for this specific mass + mass_global_bounds = get_outlier_thres(all_vals, prob) + push!(mass_bounds, mass_global_bounds) + + # Normalize each file's slice for this mass using its specific bounds + for file_idx in 1:n_files + norm_img[file_idx, mass_idx] = set_pixel_depth(slices[file_idx, mass_idx], mass_global_bounds, bins) + end + end + + return (norm_img=norm_img, bounds=mass_bounds) # bounds is now a vector, one per mass +end + +function median_filter(img) + # 3x3 median filter implementation + return mapwindow(median, img, (3, 3)) +end + +# ============================================================================ +# +# +# Analysis and Visualization +# +# ============================================================================ + +""" + display_statistics(slices) + +Calculates and prints key statistics for each slice. +""" +function display_statistics(slices, names, masses) + if isempty(slices) + @warn "Cannot display statistics for empty slice list." + return nothing + end + + n_files, n_masses = size(slices) + #stats_to_calc = Dict("Mean" => mean, "Max" => maximum, "Min" => minimum, "Sum" => sum, "Std" => std) + stats_to_calc = Dict("Mean" => mean) + all_dfs = Dict{String, DataFrame}() + + for (stat_name, stat_func) in stats_to_calc + # Create a matrix to hold the statistic for each slice + stat_matrix = zeros(Float64, n_files, n_masses) + for i in 1:n_files, j in 1:n_masses + flat_slice = vec(slices[i, j]) + if !isempty(flat_slice) + stat_matrix[i, j] = stat_func(flat_slice) + end + end + + # Create the DataFrame with masses as column headers + df = DataFrame(stat_matrix, Symbol.(masses)) + # Insert the file names as the first column + insertcols!(df, 1, :Data => names) + mz_row = ["m/z"; masses...] + push!(df, mz_row) + + println("\n--- Statistics: $(stat_name) ---") + println(df) + all_dfs[stat_name] = df + end + return all_dfs +end + +function plot_slices(slices, names, masses, output_dir; stage_name, bins=256, dpi=150, global_bounds=nothing) + n_files, n_masses = size(slices) + + mkpath(output_dir) + + # If global_bounds is provided, it should now be a VECTOR of bounds (one per mass) + # If not provided, calculate per-mass bounds + if global_bounds === nothing + global_bounds = [] + for mass_idx in 1:n_masses + mass_slices = [slices[i, mass_idx] for i in 1:n_files] + all_vals = reduce(vcat, [vec(s) for s in mass_slices]) + filter!(isfinite, all_vals) + mass_bounds = isempty(all_vals) ? (0.0, 1.0) : extrema(all_vals) + push!(global_bounds, mass_bounds) + end + end + + # With Makie, we define a Figure and a layout. + fig = Figure(size = (400 * n_masses, 330 * n_files)) # Adjusted size calculation + + # Add a title for the entire figure. + Label(fig[0, 1:(2*n_masses)], "Image Slices - $(stage_name)", fontsize=24, font=:bold, tellwidth=false, padding=(0,0,10,0)) + + for file_idx in 1:n_files + for mass_idx in 1:n_masses + img = slices[file_idx, mass_idx] + mass = masses[mass_idx] + name = names[file_idx] + mass_global_bounds = global_bounds[mass_idx] # Get bounds for THIS specific mass + + # --- Calculate tick properties for THIS mass --- + min_val, max_val = mass_global_bounds + levels = range(min_val, stop=max_val, length=bins + 1) + level_range = levels[end] - levels[1] + + if level_range == 0 + levels = range(min_val - 0.1, stop=max_val + 0.1, length=bins + 1) + level_range = 0.2 + end + + exponent = level_range > 0 ? floor(log10(level_range)) / 3 : 0 + scale = 10^(3 * exponent) + scaled_levels = levels ./ scale + + format_num = level_range > 0 ? floor(log10(level_range)) % 3 : 0 + labels = if format_num == 0 + [@sprintf("%3.2f", lvl) for lvl in scaled_levels] + elseif format_num == 1 + [@sprintf("%3.2f", lvl) for lvl in scaled_levels] + else + [@sprintf("%3.2f", lvl) for lvl in scaled_levels] + end + + divisors = 2:7 + remainders = (bins - 1) .% divisors + best_divisor = divisors[findlast(x -> x == minimum(remainders), remainders)] + tick_indices = round.(Int, range(1, stop=bins + 1, length=best_divisor + 1)) + + if !(1 in tick_indices) + pushfirst!(tick_indices, 1) + end + if !((bins + 1) in tick_indices) + push!(tick_indices, bins + 1) + end + unique!(sort!(tick_indices)) + + tick_positions = levels[tick_indices] + tick_labels = labels[tick_indices] + # --- End of mass-specific tick calculation --- + + # Create an Axis for the heatmap + ax = CairoMakie.Axis(fig[file_idx, 2*mass_idx-1], + aspect=DataAspect(), + title=@sprintf("%s\nm/z: %.2f", basename(name), mass), + titlesize=14 + ) + hidedecorations!(ax) + + # Use mass-specific bounds for colorrange + hm = heatmap!(ax, transpose(img), + colormap=cgrad(ColorSchemes.viridis, bins), + colorrange=mass_global_bounds # ← This is mass-specific + ) + + # Add a colorbar with mass-specific scale + cb = Colorbar(fig[file_idx, 2*mass_idx], hm, + label=(scale == 1 ? "" : "×10^$(round(Int, 3 * exponent))"), + labelpadding=2, + labelsize=12, + ticks=(tick_positions, tick_labels), + ticklabelsize=10 + ) + colsize!(fig.layout, 2*mass_idx, 30) + end + end + + colgap!(fig.layout, 5) + rowgap!(fig.layout, 10) + + # Save in multiple formats. + formats = ["png", "pdf"] + + for fmt in formats + filename = "$(stage_name)_overview.$(fmt)" + save_path = joinpath(output_dir, filename) + save(save_path, fig, px_per_unit = dpi / 96.0) + @info "Saved $(fmt) overview plot to $save_path" + end + return fig +end + +# ============================================================================= +# +# +# Main Workflow +# +# ============================================================================ + +function load_config(path) + config = Dict{String, Any}() + if !isfile(path) + @warn "Config file not found at $(path). Using default parameters." + # Define defaults here in case the file is missing + config["masses"] = [100] + config["tolerance"] = 0.1 + config["color_depth"] = 255 + config["outlier_prob"] = 0.98 + return config + end + + for line in eachline(path) + line = strip(line) + if isempty(line) || startswith(line, "#") + continue # Skip empty lines and comments + end + parts = split(line, '=', limit=2) + if length(parts) != 2 + @warn "Skipping malformed line in config: $(line)" + continue + end + key = strip(parts[1]) + value_str = strip(parts[2]) + + try + if key == "masses" + values = [parse(Float64, s) for s in split(value_str, ',')] + config[key] = values + elseif key == "tolerance" + config[key] = parse(Float64, value_str) + elseif key == "color_depth" + config[key] = parse(Int, value_str) + elseif key == "outlier_prob" + config[key] = parse(Float64, value_str) + end + catch e + @error "Could not parse value for key '$(key)': $(value_str)" + end + end + return config +end diff --git a/src/imzML_old.jl b/src/imzML_old.jl new file mode 100644 index 0000000..dbb1480 --- /dev/null +++ b/src/imzML_old.jl @@ -0,0 +1,246 @@ +# include( "Common.jl" ); +# ******************************************************************* +# Load Spectra and return a matrix +# fileName: Full name path +# ******************************************************************* +""" + LoadImzml( fileName ) + +Load an imzML file as a matrix. Each column stores x-pixel position, + y-pixel position, x-axis data and y-axis data. + +# Arguments +* `fileName`: Full path name of the imzML file + +# Examples +```julia +# Load DESI MSI Carcinoma image data +spectra = LoadImzml( "80TopL, 50TopR, 70BottomL, 60BottomR-centroid.imzML" ) +size( spectra ) +(4, 18632) +``` +""" +function LoadImzml( fileName ) + + # Open file handles + if !isfile(fileName) + error("provided path is not a file") + end + + if endswith(fileName, ".imzML") + stream = open(fileName) + hIbd = open(replace(fileName, ".imzML" => ".ibd")) + else + stream = open(fileName * ".imzML") + hIbd = open(fileName * ".ibd") + end + + # Get axes types and image dimensions + axis = AxesConfigImg( stream ) + imgDim = GetImgDimensions( stream ) + format = [ axis[1].Format, axis[2].Format ] + + # Locate spectrum attributes + start = position( stream ) + attr = GetSpectrumAttributes( stream, hIbd ) + + # Load spectra + seek( stream, start ) + spectra = LoadImgData( stream, hIbd, attr, imgDim[3], format ) + + close( stream ) + close( hIbd ) + return spectra + +end + + +# ******************************************************************* +# Get axes value type +# ******************************************************************* +function AxesConfigImg( stream ) + + # Locate which axis is defined at first + tag = FindTag( stream, r"^\s*<(referenceableParamGroup )" ) + value = GetAttribute( tag.captures[1], "intensityArray" ) + order = 1 + ( value !== nothing ) + + # Read first axis configuration + axis = Array{ SpecDim, 1 }( undef, 2 ) + axis[ order ] = ConfigureSpecDim( stream ) + + # Read second axis configuration + FindTag( stream, r"^\s*<(referenceableParamGroup )" ) + axis[ xor( order,3 ) ] = ConfigureSpecDim( stream ) + return axis + +end + + +# ******************************************************************* +# Get vector's storage options and image dimensions +# ******************************************************************* +function GetImgDimensions( stream ) + + # Looks for "scanSettings" tag + # FindImgTag( stream, "scanSettings" ) + FindTag( stream, r"^\s*<(scanSettings )" ) + + # Initial values for dimension retrieve + n = 2 + dim = [ 0, 0, 0 ] + currLine = "" + matchInfo = RegexMatch + + while( n > 0 ) + + # Next field + currLine = readline( stream ) + matchInfo = match( r"^\s*<(cvParam)", currLine ) + index = length( matchInfo.captures[1] ) + matchInfo = GetAttribute( currLine[index:end], "accession" ) + + # Get axis identity + if( matchInfo.captures[1] == "IMS:1000042" # max X + || matchInfo.captures[1] == "IMS:1000043" ) # max Y + + # Read dimension's pixels + axis = matchInfo.captures[1][end] - '1' + index += matchInfo.offsets[1] + length( matchInfo.captures[1] ) + matchInfo = GetAttribute( currLine[index:end], "value" ) + dim[axis] = parse( Int32, matchInfo.captures[1] ) + n -= 1 + end + end + + # Load stored spectra counter + matchInfo = FindTag( stream, r"^\s*", line); break; end + if occursin("") + if binary_tag === nothing; error("Cannot find binary tag"); end + binary_offset = position(stream) + + find_tag(stream, r"") # Position for next call + + return SpectrumAsset(params.format, params.is_compressed, binary_offset, encoded_length, params.axis_type) +end + +function parse_spectrum_metadata(stream, offset::Int64) + seek(stream, offset) + + id_match = find_tag(stream, r"(\d+)", line) + if m !== nothing + push!(offsets, parse(Int64, m.captures[1])) + elseif occursin("", line) || occursin("", line) + break + end + end + return offsets +end + +function load_mzml_lazy(file_path::String) + stream = open(file_path) + + try + seekend(stream) + skip(stream, -4096) # Read last 4KB + footer = read(stream, String) + + index_offset_match = match(r"(\d+)", footer) + if index_offset_match === nothing + error("Could not find . File may not be an indexed mzML.") + end + + index_offset = parse(Int64, index_offset_match.captures[1]) + seek(stream, index_offset) + + if find_tag(stream, r"(?\d+)", footer ) + + if index !== nothing + + # Jump to index table + seek( stream, parse( Int64, index.captures[1] ) ) + + # Find first "index" tag + index = FindTag( stream, r"(?\d+)<", String( raw ) ) + + # Exit condition + if index === nothing + break + end + + # Save index + used += 1 + unused -= 1 + entry[used] = parse( Int64, index.captures[1] ) + + # Grow list when needed + if unused == 0 + append!( entry, zeros( Int64, 8 ) ) + unused = 8 + end + + end + + return entry[1:used] + +end + + +# ******************************************************************* +# Read vector data +# stream: Source file stream +# ******************************************************************* +function ReadVector( stream, axis ) + + # Read base64 vector + base64Vec = FindTag( stream, r"([^<]+)<" ) + base64Vec = Vector{ UInt8 }( base64Vec.captures[1] ) + + # Fill a IO stream with decoded data + if axis.Packed == 1 + io = IOBuffer( Libz.inflate( Decode64( base64Vec ) ) ) + else + io = IOBuffer( Decode64( base64Vec ) ) + end + + # Convert IO stream content as dataType vector + nElem = Int32( io.size / sizeof( axis.Format ) ) + out = Array{ axis.Format }( undef, nElem ) + read!( io, out ) + + return out + +end + + +# ******************************************************************* +# Get the decoding options for the first vector in spectrum +# stream: Valid file stream +# lastTag: Last tag to skip i.e. "binaryDataArray" +# ******************************************************************* +function AxisConfig( stream ) + + offset = position( stream ) + skip = 0 + while true + + # Load line + raw = readline( stream ) + tag = match( r"^\s*<([^\s]+)", raw ) + start = length( tag.match ) + 1 + + # Is a "cvParam" tag? + if tag.captures[1] == "cvParam" + + # Get accesion field value + attribute = GetAttribute( raw[ start:end ], "accession" ) + + # Subtract file name length from skip field + if attribute.captures[1] != "MS:1000796" + continue + else + start += attribute.offset + length( attribute.match ) + value = GetAttribute( raw[ start:end ], "value" ) + skip += length( value.captures[1] ) + continue + end + + # Exit condition + elseif tag.captures[1] == "binaryDataArray" + value = GetAttribute( raw[ start:end ], "encodedLength" ) + offset = position( stream ) - offset + front = ConfigureSpecDim( stream ) + front.Skip += offset - length( value.captures[1] ) - skip + return ( front ) + + end + end +end + + +# ******************************************************************* +# Load Spectra and return a matrix +# stream: Source file stream +# ******************************************************************* +function LoadSpectra( stream, offset ) + + # Move file pointer towards first spectrum tag + seek( stream, offset[1] ) + + # Decodes axis information and skip counter + firstAxis = AxisConfig( stream ) + secondAxis = AxisConfig( stream ) + + # Allocates memory for spectra storage + specCount = length( offset ) + spectrum = Array{ Any }( undef, ( 2, specCount ) ) + index = 1 + order = 1 + ( firstAxis.Axis == 2 ) + + for i in 1:specCount + + # Load first vector + seek( stream, offset[ i ] ) + skip( stream, firstAxis.Skip ) + spectrum[ order, i ] = ReadVector( stream, firstAxis ) + + # Load second vector + skip( stream, secondAxis.Skip ) + spectrum[ xor( order,3 ), i ] = ReadVector( stream, secondAxis ) + end + + return( spectrum ) + +end + + +# ******************************************************************* +# Decoding base64 blocks +# ******************************************************************* + +# Decoding exchange table +base64dec = Vector{UInt8}( [ + 62, # (01) + -> 43d, 0x2B + 0, # (02) , + 0, # (03) - + 0, # (04) . + 63, # (05) / + 52, # (06) 0 + 53, # (07) 1 + 54, # (08) 2 + 55, # (09) 3 + 56, # (10) 4 + 57, # (11) 5 + 58, # (12) 6 + 59, # (13) 7 + 60, # (14) 8 + 61, # (15) 9 + 0, # (16) : + 0, # (17) ; + 0, # (18) < + 0, # (19) = + 0, # (20) > + 0, # (21) ? + 0, # (22) @ + 0, # (23) A + 1, # (24) B + 2, # (25) C + 3, # (26) D + 4, # (27) E + 5, # (28) F + 6, # (29) G + 7, # (40) H + 8, # (41) I + 9, # (42) J + 10, # (43) K + 11, # (44) L + 12, # (45) M + 13, # (46) N + 14, # (47) O + 15, # (48) P + 16, # (49) Q + 17, # (40) R + 18, # (41) S + 19, # (42) T + 20, # (43) U + 21, # (44) V + 22, # (45) W + 23, # (46) X + 24, # (47) Y + 25, # (48) Z + 0, # (49) [ + 0, # (50) \ + 0, # (51) ] + 0, # (52) ^ + 0, # (53) _ + 0, # (54) ` + 26, # (55) a + 27, # (56) b + 28, # (57) c + 29, # (58) d + 30, # (59) e + 31, # (60) f + 32, # (61) g + 33, # (62) h + 34, # (63) i + 35, # (64) j + 36, # (65) k + 37, # (66) l + 38, # (67) m + 39, # (68) n + 40, # (69) o + 41, # (70) p + 42, # (71) q + 43, # (72) r + 44, # (73) s + 45, # (74) t + 46, # (75) u + 47, # (76) v + 48, # (77) w + 49, # (78) x + 50, # (79) y + 51, # (80) z +] ) + +# base64 decoding +function DecodeTriplet( data, base64 ) + + # Character to binary equivalent + data = data .- 0x2A + data[1] = base64[ data[1] ] + data[2] = base64[ data[2] ] + data[3] = base64[ data[3] ] + data[4] = base64[ data[4] ] + + # Get Triplet + decode = Array{UInt8}( undef,3 ) + decode[1] = data[1] << 2 + data[2] >> 4 + decode[2] = data[2] << 4 + data[3] >> 2 + decode[3] = data[3] << 6 + data[4] + return decode + +end + + +function Decode64( data ) + + # Reserve uninitialized memory + block = divrem( sizeof( data ), 4 ) + decode = Array{UInt8}( undef, 3*block[1] ) + + # Recover binary from Chars + index = 1 + first = 1 + last = 3 + + for i in 1:block[1] + decode[first:last] = DecodeTriplet( data[index:index+3], base64dec ) + index += 4 + first += 3 + last += 3 + end + + # Adjust block size + if data[end-1] == 0x3D + decode = decode[1:end-2] + elseif data[end] == 0x3D + decode = decode[1:end-1] + end + + return decode + +end diff --git a/start_MSI_GUI.jl b/start_MSI_GUI.jl index 0d69ef8..14036e2 100644 --- a/start_MSI_GUI.jl +++ b/start_MSI_GUI.jl @@ -1,7 +1,11 @@ using Pkg sTime=time() Pkg.activate(".") -Pkg.instantiate() +# Only instantiate in development mode +if get(ENV, "GENIE_ENV", "dev") != "prod" + @info "Development environment detected. Instantiating packages..." + Pkg.instantiate() +end Pkg.gc() using Genie From cd1115368f230fa4d71b6c79d012dd777ba4c4dc Mon Sep 17 00:00:00 2001 From: Pixelguy14 Date: Mon, 29 Sep 2025 10:57:11 -0600 Subject: [PATCH 02/19] finished test script, included instructions in test/readme.md, the test performs mzml, imzml and conversion tests, along with throwing reference plots and images which result from the files --- .gitignore | 1 + Manifest.toml | 1109 ++++++++++++++++++++++++------------------ Project.toml | 10 + app.jl | 6 +- src/Bitmap.jl | 399 --------------- src/Common.jl | 94 ---- src/DataAccess.jl | 82 ---- src/MSIData.jl | 369 ++++++++++++++ src/MSI_src.jl | 33 ++ src/MzmlConverter.jl | 629 +++++++++++++++++++----- src/ParserHelpers.jl | 15 +- src/imzML.jl | 347 +++++++------ src/imzML_old.jl | 246 ---------- src/mzML.jl | 219 +++++---- src/mzML_old.jl | 350 ------------- test/readme.md | 21 + test/run_tests.jl | 299 ++++++++++++ 17 files changed, 2145 insertions(+), 2084 deletions(-) delete mode 100644 src/Bitmap.jl delete mode 100644 src/Common.jl delete mode 100644 src/DataAccess.jl create mode 100644 src/MSIData.jl create mode 100644 src/MSI_src.jl delete mode 100644 src/imzML_old.jl delete mode 100644 src/mzML_old.jl create mode 100644 test/readme.md create mode 100644 test/run_tests.jl diff --git a/.gitignore b/.gitignore index 1905238..8421c56 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ public/css/imgOver.png !public/css/LABI_logo.png log/* R_original_scripts/ +test/results/ diff --git a/Manifest.toml b/Manifest.toml index 319d86f..ff5e2fb 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,14 +1,14 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.3" +julia_version = "1.11.7" manifest_format = "2.0" -project_hash = "e1be01554a547da63b1504fb440ab6cc4855bc9d" +project_hash = "896c16630591d2732da8e3315c4b64c618fd5ebb" [[deps.ATK_jll]] deps = ["Artifacts", "Glib_jll", "JLLWrappers", "Libdl"] -git-tree-sha1 = "a2ecb68d240333fe63bea1965b71884e98c2d0f0" +git-tree-sha1 = "c0a10eaf3a7e1e909ef8fe45971a73524bfce1bf" uuid = "7b86fcea-f67b-53e1-809c-8f1719c154e8" -version = "2.38.0+0" +version = "2.38.1+0" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -28,12 +28,13 @@ version = "0.4.5" [[deps.Adapt]] deps = ["LinearAlgebra", "Requires"] -git-tree-sha1 = "50c3c56a52972d78e8be9fd135bfb91c9574c140" +git-tree-sha1 = "7e35fca2bdfba44d797c53dfe63a51fabf39bfc0" uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "4.1.1" -weakdeps = ["StaticArrays"] +version = "4.4.0" +weakdeps = ["SparseArrays", "StaticArrays"] [deps.Adapt.extensions] + AdaptSparseArraysExt = "SparseArrays" AdaptStaticArraysExt = "StaticArrays" [[deps.AdaptivePredicates]] @@ -71,9 +72,9 @@ version = "0.4.0" [[deps.ArrayInterface]] deps = ["Adapt", "LinearAlgebra"] -git-tree-sha1 = "017fcb757f8e921fb44ee063a7aafe5f89b86dd1" +git-tree-sha1 = "dbd8c3bbbdbb5c2778f85f4422c39960eac65a42" uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" -version = "7.18.0" +version = "7.20.0" [deps.ArrayInterface.extensions] ArrayInterfaceBandedMatricesExt = "BandedMatrices" @@ -83,6 +84,7 @@ version = "7.18.0" ArrayInterfaceChainRulesCoreExt = "ChainRulesCore" ArrayInterfaceChainRulesExt = "ChainRules" ArrayInterfaceGPUArraysCoreExt = "GPUArraysCore" + ArrayInterfaceMetalExt = "Metal" ArrayInterfaceReverseDiffExt = "ReverseDiff" ArrayInterfaceSparseArraysExt = "SparseArrays" ArrayInterfaceStaticArraysCoreExt = "StaticArraysCore" @@ -96,6 +98,7 @@ version = "7.18.0" ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" @@ -119,14 +122,19 @@ version = "1.1.0" [[deps.AxisArrays]] deps = ["Dates", "IntervalSets", "IterTools", "RangeArrays"] -git-tree-sha1 = "16351be62963a67ac4083f748fdb3cca58bfd52f" +git-tree-sha1 = "4126b08903b777c88edf1754288144a0492c05ad" uuid = "39de3d68-74b9-583c-8d2d-e117c070f3a9" -version = "0.4.7" +version = "0.4.8" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" +[[deps.BaseDirs]] +git-tree-sha1 = "bca794632b8a9bbe159d56bf9e31c422671b35e0" +uuid = "18cc8868-cbac-4acf-b575-c8ff214dc66f" +version = "1.3.2" + [[deps.BitFlags]] git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" @@ -144,10 +152,10 @@ uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" version = "1.2.2" [[deps.Bzip2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "8873e196c2eb87962a2048b3b8e08946535864a1" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1b96ea4a01afe0ea4090c5c8039690672dd13f2e" uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" -version = "1.0.8+4" +version = "1.0.9+0" [[deps.CEnum]] git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" @@ -155,15 +163,21 @@ uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" version = "0.5.0" [[deps.CPUSummary]] -deps = ["CpuId", "IfElse", "PrecompileTools", "Static"] -git-tree-sha1 = "5a97e67919535d6841172016c9530fd69494e5ec" +deps = ["CpuId", "IfElse", "PrecompileTools", "Preferences", "Static"] +git-tree-sha1 = "f3a21d7fc84ba618a779d1ed2fcca2e682865bab" uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9" -version = "0.2.6" +version = "0.2.7" [[deps.CRC32c]] uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" version = "1.11.0" +[[deps.CRlibm]] +deps = ["CRlibm_jll"] +git-tree-sha1 = "66188d9d103b92b6cd705214242e27f5737a1e5e" +uuid = "96374032-68de-5a5b-8d9e-752f78720389" +version = "1.0.2" + [[deps.CRlibm_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "e329286945d0cfc04456972ea732551869af1cfc" @@ -176,6 +190,12 @@ git-tree-sha1 = "0157e592151e39fa570645e2b2debcdfb8a0f112" uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f" version = "3.4.3" +[[deps.CSV]] +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] +git-tree-sha1 = "deddd8725e5e1cc49ee205a1964256043720a6c3" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.10.15" + [[deps.Cairo]] deps = ["Cairo_jll", "Colors", "Glib_jll", "Graphics", "Libdl", "Pango_jll"] git-tree-sha1 = "71aa551c5c33f1a4415867fe06b7844faadb0ae9" @@ -190,9 +210,9 @@ version = "0.13.1" [[deps.Cairo_jll]] deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] -git-tree-sha1 = "009060c9a6168704143100f36ab08f06c2af4642" +git-tree-sha1 = "fde3bf89aead2e723284a8ff9cdf5b551ed700e8" uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" -version = "1.18.2+1" +version = "1.18.5+0" [[deps.CatIndices]] deps = ["CustomUnitRanges", "OffsetArrays"] @@ -202,14 +222,31 @@ version = "0.2.2" [[deps.ChainRulesCore]] deps = ["Compat", "LinearAlgebra"] -git-tree-sha1 = "1713c74e00545bfe14605d2a2be1712de8fbcb58" +git-tree-sha1 = "e4c6a16e77171a5f5e25e9646617ab1c276c5607" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "1.25.1" +version = "1.26.0" weakdeps = ["SparseArrays"] [deps.ChainRulesCore.extensions] ChainRulesCoreSparseArraysExt = "SparseArrays" +[[deps.ChunkCodecCore]] +git-tree-sha1 = "51f4c10ee01bda57371e977931de39ee0f0cdb3e" +uuid = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" +version = "1.0.0" + +[[deps.ChunkCodecLibZlib]] +deps = ["ChunkCodecCore", "Zlib_jll"] +git-tree-sha1 = "cee8104904c53d39eb94fd06cbe60cb5acde7177" +uuid = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" +version = "1.0.0" + +[[deps.ChunkCodecLibZstd]] +deps = ["ChunkCodecCore", "Zstd_jll"] +git-tree-sha1 = "34d9873079e4cb3d0c62926a225136824677073f" +uuid = "55437552-ac27-4d47-9aa3-63184e8fd398" +version = "1.0.0" + [[deps.CloseOpenIntervals]] deps = ["Static", "StaticArrayInterface"] git-tree-sha1 = "05ba0d07cd4fd8b7a39541e31a7b0254704ea581" @@ -224,15 +261,15 @@ version = "0.15.8" [[deps.CodeTracking]] deps = ["InteractiveUtils", "UUIDs"] -git-tree-sha1 = "7eee164f122511d3e4e1ebadb7956939ea7e1c77" +git-tree-sha1 = "980f01d6d3283b3dbdfd7ed89405f96b7256ad57" uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" -version = "1.3.6" +version = "2.0.1" [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.6" +version = "0.7.8" [[deps.ColorBrewer]] deps = ["Colors", "JSON"] @@ -242,9 +279,9 @@ version = "0.4.1" [[deps.ColorSchemes]] deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "PrecompileTools", "Random"] -git-tree-sha1 = "26ec26c98ae1453c692efded2b17e15125a5bea1" +git-tree-sha1 = "b0fd3f56fa442f81e0a47815c92245acfaaa4e34" uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" -version = "3.28.0" +version = "3.31.0" [[deps.ColorTypes]] deps = ["FixedPointNumbers", "Random"] @@ -270,9 +307,9 @@ version = "0.12.11" [[deps.CommonMark]] deps = ["Crayons", "PrecompileTools"] -git-tree-sha1 = "3faae67b8899797592335832fccf4b3c80bb04fa" +git-tree-sha1 = "5fdf00d1979fd4883b44b754fc3423175c9504b4" uuid = "a80b9123-70ca-4bc0-993e-6e3bcb318db6" -version = "0.8.15" +version = "0.8.16" [[deps.CommonWorldInvalidations]] git-tree-sha1 = "ae52d1c52048455e85a387fbee9be553ec2b68d0" @@ -281,14 +318,19 @@ version = "1.0.0" [[deps.Compat]] deps = ["TOML", "UUIDs"] -git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +git-tree-sha1 = "0037835448781bb46feb39866934e243886d756a" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.16.0" +version = "4.18.0" weakdeps = ["Dates", "LinearAlgebra"] [deps.Compat.extensions] CompatLinearAlgebraExt = "LinearAlgebra" +[[deps.Compiler]] +git-tree-sha1 = "382d79bfe72a406294faca39ef0c3cef6e6ce1f1" +uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" +version = "0.1.1" + [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" @@ -301,9 +343,9 @@ version = "0.3.2" [[deps.ConcurrentUtilities]] deps = ["Serialization", "Sockets"] -git-tree-sha1 = "f36e5e8fdffcb5646ea5da81495a5a7566005127" +git-tree-sha1 = "d9d26935a0bcffc87d2613ce14c527c99fc543fd" uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" -version = "2.4.3" +version = "2.5.0" [[deps.Conda]] deps = ["Downloads", "JSON", "VersionParsing"] @@ -318,9 +360,9 @@ uuid = "5218b696-f38b-4ac9-8b61-a12ec717816d" version = "0.17.6" [[deps.ConstructionBase]] -git-tree-sha1 = "76219f1ed5771adbb096743bff43fb5fdd4c1157" +git-tree-sha1 = "b4b092499347b18a015186eae3042f72267106cb" uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" -version = "1.5.8" +version = "1.6.0" weakdeps = ["IntervalSets", "LinearAlgebra", "StaticArrays"] [deps.ConstructionBase.extensions] @@ -335,9 +377,9 @@ version = "0.6.3" [[deps.CoordinateTransformations]] deps = ["LinearAlgebra", "StaticArrays"] -git-tree-sha1 = "f9d7112bfff8a19a3a4ea4e03a8e6a91fe8456bf" +git-tree-sha1 = "a692f5e257d332de1e554e4566a4e5a8a72de2b2" uuid = "150eb455-5306-5404-9cee-2592286d6298" -version = "0.6.3" +version = "0.6.4" [[deps.CpuId]] deps = ["Markdown"] @@ -368,9 +410,9 @@ version = "1.7.0" [[deps.DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +git-tree-sha1 = "4e1fe97fdaed23e9dc21d4d664bea76b65fc50a0" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.20" +version = "0.18.22" [[deps.DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" @@ -384,9 +426,9 @@ version = "1.11.0" [[deps.Dbus_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl"] -git-tree-sha1 = "fc173b380865f70627d7dd1190dc2fce6cc105af" +git-tree-sha1 = "473e9afc9cf30814eb67ffa5f2db7df82c3ad9fd" uuid = "ee1fde0b-3d02-5ea6-8484-8dfef6360eab" -version = "1.14.10+0" +version = "1.16.2+0" [[deps.DelaunayTriangulation]] deps = ["AdaptivePredicates", "EnumX", "ExactPredicates", "Random"] @@ -418,9 +460,9 @@ version = "1.11.0" [[deps.Distributions]] deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] -git-tree-sha1 = "03aa5d44647eaec98e1920635cdfed5d5560a8b9" +git-tree-sha1 = "3e6d038b77f22791b8e3472b7c633acea1ecac06" uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.25.117" +version = "0.25.120" [deps.Distributions.extensions] DistributionsChainRulesCoreExt = "ChainRulesCore" @@ -433,10 +475,9 @@ version = "0.25.117" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[deps.DocStringExtensions]] -deps = ["LibGit2"] -git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.3" +version = "0.9.5" [[deps.DotEnv]] deps = ["PrecompileTools"] @@ -456,9 +497,9 @@ uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5" version = "2.2.4+0" [[deps.EnumX]] -git-tree-sha1 = "bdb1942cd4c45e3c678fd11569d5cccd80976237" +git-tree-sha1 = "bddad79635af6aec424f53ed8aad5d7555dc6f00" uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56" -version = "1.0.4" +version = "1.0.5" [[deps.EpollShim_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -468,9 +509,9 @@ version = "0.0.20230411+1" [[deps.ExactPredicates]] deps = ["IntervalArithmetic", "Random", "StaticArrays"] -git-tree-sha1 = "b3f2ff58735b5f024c392fde763f29b057e4b025" +git-tree-sha1 = "83231673ea4d3d6008ac74dc5079e77ab2209d8f" uuid = "429591f6-91af-11e9-00e2-59fbe8cec110" -version = "2.2.8" +version = "2.2.9" [[deps.ExceptionUnwrapping]] deps = ["Test"] @@ -480,42 +521,42 @@ version = "0.1.11" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "e51db81749b0777b2147fbe7b783ee79045b8e99" +git-tree-sha1 = "7bb1361afdb33c7f2b085aa49ea8fe1b0fb14e58" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.4+3" +version = "2.7.1+0" [[deps.ExpressionExplorer]] -git-tree-sha1 = "71d0768dd78ad62d3582091bf338d98af8bbda67" +git-tree-sha1 = "4a8c0a9eebf807ac42f0f6de758e60a20be25ffb" uuid = "21656369-7473-754a-2065-74616d696c43" -version = "1.1.1" +version = "1.1.3" [[deps.ExproniconLite]] -git-tree-sha1 = "4c9ed87a6b3cd90acf24c556f2119533435ded38" +git-tree-sha1 = "c13f0b150373771b0fdc1713c97860f8df12e6c2" uuid = "55351af7-c7e9-48d6-89ff-24e801d99491" -version = "0.10.13" +version = "0.10.14" [[deps.Extents]] -git-tree-sha1 = "063512a13dbe9c40d999c439268539aa552d1ae6" +git-tree-sha1 = "b309b36a9e02fe7be71270dd8c0fd873625332b4" uuid = "411431e0-e8b7-467b-b5e0-f676ba4f2910" -version = "0.1.5" +version = "0.1.6" [[deps.EzXML]] deps = ["Printf", "XML2_jll"] -git-tree-sha1 = "380053d61bb9064d6aa4a9777413b40429c79901" +git-tree-sha1 = "680a39c9aadce7c721b68d979e66dc65d2021aa6" uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" -version = "1.2.0" +version = "1.2.2" [[deps.FFMPEG]] deps = ["FFMPEG_jll"] -git-tree-sha1 = "53ebe7511fa11d33bec688a9178fac4e49eeee00" +git-tree-sha1 = "83dc665d0312b41367b7263e8a4d172eac1897f4" uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" -version = "0.4.2" +version = "0.4.4" [[deps.FFMPEG_jll]] -deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "PCRE2_jll", "Pkg", "Zlib_jll", "libaom_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] -git-tree-sha1 = "74faea50c1d007c85837327f6775bea60b5492dd" +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "PCRE2_jll", "Zlib_jll", "libaom_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] +git-tree-sha1 = "eaa040768ea663ca695d442be1bc97edfe6824f2" uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" -version = "4.4.2+2" +version = "6.1.3+0" [[deps.FFTViews]] deps = ["CustomUnitRanges", "FFTW"] @@ -524,22 +565,22 @@ uuid = "4f61f5a4-77b1-5117-aa51-3ab5ef4ef0cd" version = "0.3.2" [[deps.FFTW]] -deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] -git-tree-sha1 = "7de7c78d681078f027389e067864a8d53bd7c3c9" +deps = ["AbstractFFTs", "FFTW_jll", "Libdl", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] +git-tree-sha1 = "97f08406df914023af55ade2f843c39e99c5d969" uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "1.8.1" +version = "1.10.0" [[deps.FFTW_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "4d81ed14783ec49ce9f2e168208a12ce1815aa25" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6d6219a004b8cf1e0b4dbe27a2860b8e04eba0be" uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" -version = "3.3.10+3" +version = "3.3.11+0" [[deps.FileIO]] deps = ["Pkg", "Requires", "UUIDs"] -git-tree-sha1 = "2dd20384bf8c6d411b5c7370865b1e9b26cb2ea3" +git-tree-sha1 = "b66970a70db13f45b7e57fbda1736e1cf72174ea" uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" -version = "1.16.6" +version = "1.17.0" weakdeps = ["HTTP"] [deps.FileIO.extensions] @@ -553,9 +594,9 @@ version = "0.8.3" [[deps.FilePathsBase]] deps = ["Compat", "Dates"] -git-tree-sha1 = "7878ff7172a8e6beedd1dea14bd27c3c6340d361" +git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2" uuid = "48062228-2e41-5def-b9a4-89aafe57970f" -version = "0.9.22" +version = "0.9.24" weakdeps = ["Mmap", "Test"] [deps.FilePathsBase.extensions] @@ -568,9 +609,9 @@ version = "1.11.0" [[deps.FillArrays]] deps = ["LinearAlgebra"] -git-tree-sha1 = "6a70198746448456524cb442b8af316927ff3e1a" +git-tree-sha1 = "173e4d8f14230a7523ae11b9a3fa9edb3e0efd78" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "1.13.0" +version = "1.14.0" weakdeps = ["PDMats", "SparseArrays", "Statistics"] [deps.FillArrays.extensions] @@ -586,9 +627,9 @@ version = "0.8.5" [[deps.Fontconfig_jll]] deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Zlib_jll"] -git-tree-sha1 = "21fac3c77d7b5a9fc03b0ec503aa1a6392c34d2b" +git-tree-sha1 = "f85dac9a96a01087df6e3a749840015a0ca3817d" uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" -version = "2.15.0+0" +version = "2.17.1+0" [[deps.Format]] git-tree-sha1 = "9c68794ef81b08086aeb32eeaf33531668d5f5fc" @@ -603,33 +644,27 @@ version = "4.1.1" [[deps.FreeType2_jll]] deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "786e968a8d2fb167f2e4880baba62e0e26bd8e4e" +git-tree-sha1 = "2c5512e11c791d1baed2049c5652441b28fc6a31" uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" -version = "2.13.3+1" +version = "2.13.4+0" [[deps.FreeTypeAbstraction]] -deps = ["ColorVectorSpace", "Colors", "FreeType", "GeometryBasics"] -git-tree-sha1 = "d52e255138ac21be31fa633200b65e4e71d26802" +deps = ["BaseDirs", "ColorVectorSpace", "Colors", "FreeType", "GeometryBasics", "Mmap"] +git-tree-sha1 = "4ebb930ef4a43817991ba35db6317a05e59abd11" uuid = "663a7486-cb36-511b-a19d-713bb74d65c9" -version = "0.10.6" +version = "0.10.8" [[deps.FriBidi_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "846f7026a9decf3679419122b49f8a1fdb48d2d5" +git-tree-sha1 = "7a214fdac5ed5f59a22c2d9a885a16da1c74bbc7" uuid = "559328eb-81f9-559d-9380-de523a88c83c" -version = "1.0.16+0" +version = "1.0.17+0" [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" -[[deps.FuzzyCompletions]] -deps = ["REPL"] -git-tree-sha1 = "be713866335f48cfb1285bff2d0cbb8304c1701c" -uuid = "fb4132e2-a121-4a70-b8a1-d5b831dcdcc2" -version = "0.5.5" - [[deps.GLFW_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll", "libdecor_jll", "xkbcommon_jll"] git-tree-sha1 = "fcb0584ff34e25155876418979d4c8971243bb89" @@ -642,16 +677,16 @@ uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" version = "6.3.0+0" [[deps.GR]] -deps = ["Artifacts", "Base64", "DelimitedFiles", "Downloads", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Preferences", "Printf", "Random", "Serialization", "Sockets", "TOML", "Tar", "Test", "UUIDs", "p7zip_jll"] -git-tree-sha1 = "8e2d86e06ceb4580110d9e716be26658effc5bfd" +deps = ["Artifacts", "Base64", "DelimitedFiles", "Downloads", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Preferences", "Printf", "Qt6Wayland_jll", "Random", "Serialization", "Sockets", "TOML", "Tar", "Test", "p7zip_jll"] +git-tree-sha1 = "1828eb7275491981fa5f1752a5e126e8f26f8741" uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71" -version = "0.72.8" +version = "0.73.17" [[deps.GR_jll]] -deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Qt5Base_jll", "Zlib_jll", "libpng_jll"] -git-tree-sha1 = "da121cbdc95b065da07fbb93638367737969693f" +deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "FreeType2_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Qt6Base_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "27299071cc29e409488ada41ec7643e0ab19091f" uuid = "d2c73de3-f751-5644-a686-071e5b155ba9" -version = "0.72.8+0" +version = "0.73.17+0" [[deps.GTK3_jll]] deps = ["ATK_jll", "Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Libepoxy_jll", "Pango_jll", "Pkg", "Wayland_jll", "Xorg_libX11_jll", "Xorg_libXcomposite_jll", "Xorg_libXcursor_jll", "Xorg_libXdamage_jll", "Xorg_libXext_jll", "Xorg_libXfixes_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll", "Xorg_libXrender_jll", "at_spi2_atk_jll", "gdk_pixbuf_jll", "iso_codes_jll", "xkbcommon_jll"] @@ -679,9 +714,9 @@ version = "2.2.5" [[deps.GenieDevTools]] deps = ["Dates", "Genie", "RemoteREPL", "Revise", "Stipple", "StippleUI", "Tables"] -git-tree-sha1 = "21ca67d9d8a9f5676c85be3cf16337b7dcdac2db" +git-tree-sha1 = "5c84de04cde55e272e760e7ef021bb645f53bd67" uuid = "4e5d9629-6565-4102-a9ff-45c707eb060e" -version = "2.12.0" +version = "2.12.1" [[deps.GenieFramework]] deps = ["GarishPrint", "Genie", "GenieAutoReload", "GenieDevTools", "GeniePackageManager", "Reexport", "Revise", "Stipple", "StipplePivotTable", "StipplePlotly", "StippleTable", "StippleTabs", "StippleUI"] @@ -707,34 +742,29 @@ git-tree-sha1 = "5f5ffc33251d0d5f110072f7a0c07717c9453c97" uuid = "5c4fdc26-39e3-47cf-9034-e533e09961c2" version = "1.1.0" -[[deps.GeoFormatTypes]] -git-tree-sha1 = "8e233d5167e63d708d41f87597433f59a0f213fe" -uuid = "68eda718-8dee-11e9-39e7-89f7f65f511f" -version = "0.4.4" - -[[deps.GeoInterface]] -deps = ["DataAPI", "Extents", "GeoFormatTypes"] -git-tree-sha1 = "294e99f19869d0b0cb71aef92f19d03649d028d5" -uuid = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" -version = "1.4.1" - [[deps.GeometryBasics]] -deps = ["EarCut_jll", "Extents", "GeoInterface", "IterTools", "LinearAlgebra", "PrecompileTools", "Random", "StaticArrays"] -git-tree-sha1 = "c1a9c159c3ac53aa09663d8662c7277ef3fa508d" +deps = ["EarCut_jll", "Extents", "IterTools", "LinearAlgebra", "PrecompileTools", "Random", "StaticArrays"] +git-tree-sha1 = "1f5a80f4ed9f5a4aada88fc2db456e637676414b" uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326" -version = "0.5.1" +version = "0.5.10" -[[deps.Gettext_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] -git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046" -uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" -version = "0.21.0+0" + [deps.GeometryBasics.extensions] + GeometryBasicsGeoInterfaceExt = "GeoInterface" + + [deps.GeometryBasics.weakdeps] + GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" + +[[deps.GettextRuntime_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll"] +git-tree-sha1 = "45288942190db7c5f760f59c04495064eedf9340" +uuid = "b0724c58-0f36-5564-988d-3bb0596ebc4a" +version = "0.22.4+0" [[deps.Ghostscript_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "43ba3d3c82c18d88471cfd2924931658838c9d8f" +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Zlib_jll"] +git-tree-sha1 = "38044a04637976140074d0b0621c1edf0eb531fd" uuid = "61579ee1-b43e-5ca0-a5da-69d92c66a64b" -version = "9.55.0+4" +version = "9.55.1+0" [[deps.Giflib_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -743,16 +773,22 @@ uuid = "59f7168a-df46-5410-90c8-f2779963d0ec" version = "5.2.3+0" [[deps.Glib_jll]] -deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "b0036b392358c80d2d2124746c2bf3d48d457938" +deps = ["Artifacts", "GettextRuntime_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "50c11ffab2a3d50192a228c313f05b5b5dc5acb2" uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" -version = "2.82.4+0" +version = "2.86.0+0" [[deps.Glob]] git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" version = "1.3.1" +[[deps.GracefulPkg]] +deps = ["Compat", "Pkg", "TOML"] +git-tree-sha1 = "698050b04f3cc0906d0817329d6e96484bf238eb" +uuid = "828d9ff0-206c-6161-646e-6576656f7244" +version = "2.3.0" + [[deps.Graphics]] deps = ["Colors", "LinearAlgebra", "NaNMath"] git-tree-sha1 = "a641238db938fff9b2f60d08ed9030387daf428c" @@ -760,22 +796,22 @@ uuid = "a2bd30eb-e257-5431-a919-1863eab51364" version = "1.1.3" [[deps.Graphite2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "01979f9b37367603e2848ea225918a3b3861b606" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "8a6dbda1fd736d60cc477d99f2e7a042acfa46e8" uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" -version = "1.3.14+1" +version = "1.3.15+0" [[deps.Graphs]] -deps = ["ArnoldiMethod", "Compat", "DataStructures", "Distributed", "Inflate", "LinearAlgebra", "Random", "SharedArrays", "SimpleTraits", "SparseArrays", "Statistics"] -git-tree-sha1 = "1dc470db8b1131cfc7fb4c115de89fe391b9e780" +deps = ["ArnoldiMethod", "DataStructures", "Distributed", "Inflate", "LinearAlgebra", "Random", "SharedArrays", "SimpleTraits", "SparseArrays", "Statistics"] +git-tree-sha1 = "7a98c6502f4632dbe9fb1973a4244eaa3324e84d" uuid = "86223c79-3864-5bf0-83f7-82e725a168b6" -version = "1.12.0" +version = "1.13.1" [[deps.GridLayoutBase]] deps = ["GeometryBasics", "InteractiveUtils", "Observables"] -git-tree-sha1 = "dc6bed05c15523624909b3953686c5f5ffa10adc" +git-tree-sha1 = "93d5c27c8de51687a2c70ec0716e6e76f298416f" uuid = "3955a311-db13-416c-9275-1d80ed98e5e9" -version = "0.11.1" +version = "0.11.2" [[deps.Grisu]] git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2" @@ -784,15 +820,20 @@ version = "1.0.2" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "c67b33b085f6e2faf8bf79a61962e7339a81129c" +git-tree-sha1 = "ed5e9c58612c4e081aecdb6e1a479e18462e041e" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.10.15" +version = "1.10.17" [[deps.HarfBuzz_jll]] deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll"] -git-tree-sha1 = "55c53be97790242c29031e5cd45e8ac296dadda3" +git-tree-sha1 = "f923f9a774fcf3f5cb761bfa43aeadd689714813" uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" -version = "8.5.0+0" +version = "8.5.1+0" + +[[deps.HashArrayMappedTries]] +git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" +uuid = "076d061b-32b6-4027-95e0-9a2c6f6d7e74" +version = "0.2.0" [[deps.HistogramThresholding]] deps = ["ImageBase", "LinearAlgebra", "MappedArrays"] @@ -814,9 +855,9 @@ version = "0.5.0" [[deps.HypergeometricFunctions]] deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] -git-tree-sha1 = "4f0a6a9f57c0d0cc7590a6e47c0f641814917f2a" +git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec" uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" -version = "0.3.26" +version = "0.3.28" [[deps.HypertextLiteral]] deps = ["Tricks"] @@ -873,9 +914,9 @@ version = "0.2.17" [[deps.ImageFiltering]] deps = ["CatIndices", "ComputationalResources", "DataStructures", "FFTViews", "FFTW", "ImageBase", "ImageCore", "LinearAlgebra", "OffsetArrays", "PrecompileTools", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "TiledIteration"] -git-tree-sha1 = "33cb509839cc4011beb45bde2316e64344b0f92b" +git-tree-sha1 = "52116260a234af5f69969c5286e6a5f8dc3feab8" uuid = "6a3955dd-da59-5b1f-98d4-e7296123deb5" -version = "0.7.9" +version = "0.7.12" [[deps.ImageIO]] deps = ["FileIO", "IndirectArrays", "JpegTurbo", "LazyModules", "Netpbm", "OpenEXR", "PNGFiles", "QOI", "Sixel", "TiffImages", "UUIDs", "WebP"] @@ -885,15 +926,15 @@ version = "0.6.9" [[deps.ImageMagick]] deps = ["FileIO", "ImageCore", "ImageMagick_jll", "InteractiveUtils"] -git-tree-sha1 = "c5c5478ae8d944c63d6de961b19e6d3324812c35" +git-tree-sha1 = "8e64ab2f0da7b928c8ae889c514a52741debc1c2" uuid = "6218d12a-5da1-5696-b52f-db25d2ecc6d1" -version = "1.4.0" +version = "1.4.2" [[deps.ImageMagick_jll]] -deps = ["Artifacts", "Ghostscript_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "OpenJpeg_jll", "Zlib_jll", "libpng_jll"] -git-tree-sha1 = "fa01c98985be12e5d75301c4527fff2c46fa3e0e" +deps = ["Artifacts", "Bzip2_jll", "FFTW_jll", "Ghostscript_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "OpenJpeg_jll", "Zlib_jll", "Zstd_jll", "libpng_jll", "libwebp_jll", "libzip_jll"] +git-tree-sha1 = "d670e8e3adf0332f57054955422e85a4aec6d0b0" uuid = "c73af94c-d91f-53ed-93a7-00f77d67a9d7" -version = "7.1.1+1" +version = "7.1.2005+0" [[deps.ImageMetadata]] deps = ["AxisArrays", "ImageAxes", "ImageBase", "ImageCore"] @@ -903,9 +944,9 @@ version = "0.9.10" [[deps.ImageMorphology]] deps = ["DataStructures", "ImageCore", "LinearAlgebra", "LoopVectorization", "OffsetArrays", "Requires", "TiledIteration"] -git-tree-sha1 = "6f0a801136cb9c229aebea0df296cdcd471dbcd1" +git-tree-sha1 = "cffa21df12f00ca1a365eb8ed107614b40e8c6da" uuid = "787d08f9-d448-5407-9aad-5290dd7ab264" -version = "0.4.5" +version = "0.4.6" [[deps.ImageQualityIndexes]] deps = ["ImageContrastAdjustment", "ImageCore", "ImageDistances", "ImageFiltering", "LazyModules", "OffsetArrays", "PrecompileTools", "Statistics"] @@ -915,9 +956,9 @@ version = "0.3.7" [[deps.ImageSegmentation]] deps = ["Clustering", "DataStructures", "Distances", "Graphs", "ImageCore", "ImageFiltering", "ImageMorphology", "LinearAlgebra", "MetaGraphs", "RegionTrees", "SimpleWeightedGraphs", "StaticArrays", "Statistics"] -git-tree-sha1 = "b217d9ded4a95052ffc09acc41ab781f7f72c7ba" +git-tree-sha1 = "7196039573b6f312864547eb7a74360d6c0ab8e6" uuid = "80713f31-8817-5129-9cf8-209ff8fb23e1" -version = "1.8.3" +version = "1.9.0" [[deps.ImageShow]] deps = ["Base64", "ColorSchemes", "FileIO", "ImageBase", "ImageCore", "OffsetArrays", "StackViews"] @@ -927,9 +968,9 @@ version = "0.3.8" [[deps.ImageTransformations]] deps = ["AxisAlgorithms", "CoordinateTransformations", "ImageBase", "ImageCore", "Interpolations", "OffsetArrays", "Rotations", "StaticArrays"] -git-tree-sha1 = "e0884bdf01bbbb111aea77c348368a86fb4b5ab6" +git-tree-sha1 = "dfde81fafbe5d6516fb864dc79362c5c6b973c82" uuid = "02fcd773-0e25-5acc-982a-7f6622650795" -version = "0.10.1" +version = "0.10.2" [[deps.Images]] deps = ["Base64", "FileIO", "Graphics", "ImageAxes", "ImageBase", "ImageBinarization", "ImageContrastAdjustment", "ImageCore", "ImageCorners", "ImageDistances", "ImageFiltering", "ImageIO", "ImageMagick", "ImageMetadata", "ImageMorphology", "ImageQualityIndexes", "ImageSegmentation", "ImageShow", "ImageTransformations", "IndirectArrays", "IntegralArrays", "Random", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "StatsBase", "TiledIteration"] @@ -960,9 +1001,9 @@ uuid = "6d011eab-0732-4556-8808-e463c76bf3b6" version = "1.1.0" [[deps.InlineStrings]] -git-tree-sha1 = "45521d31238e87ee9f9732561bfee12d4eebd52d" +git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" -version = "1.4.2" +version = "1.4.5" [deps.InlineStrings.extensions] ArrowTypesExt = "ArrowTypes" @@ -980,9 +1021,9 @@ version = "0.1.6" [[deps.IntelOpenMP_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] -git-tree-sha1 = "0f14a5456bdc6b9731a5682f439a672750a09e48" +git-tree-sha1 = "ec1debd61c300961f98064cfb21287613ad7f303" uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" -version = "2025.0.4+0" +version = "2025.2.0+0" [[deps.InteractiveUtils]] deps = ["Markdown"] @@ -1000,27 +1041,33 @@ weakdeps = ["Unitful"] InterpolationsUnitfulExt = "Unitful" [[deps.IntervalArithmetic]] -deps = ["CRlibm_jll", "LinearAlgebra", "MacroTools", "RoundingEmulator"] -git-tree-sha1 = "ffb76d09ab0dc9f5a27edac2acec13c74a876cc6" +deps = ["CRlibm", "MacroTools", "OpenBLASConsistentFPCSR_jll", "Printf", "Random", "RoundingEmulator"] +git-tree-sha1 = "815e74f416953c348c9da1d1bc977bbc97c84e18" uuid = "d1acc4aa-44c8-5952-acd4-ba5d80a2a253" -version = "0.22.21" +version = "1.0.0" [deps.IntervalArithmetic.extensions] + IntervalArithmeticArblibExt = "Arblib" IntervalArithmeticDiffRulesExt = "DiffRules" IntervalArithmeticForwardDiffExt = "ForwardDiff" IntervalArithmeticIntervalSetsExt = "IntervalSets" + IntervalArithmeticLinearAlgebraExt = "LinearAlgebra" IntervalArithmeticRecipesBaseExt = "RecipesBase" + IntervalArithmeticSparseArraysExt = "SparseArrays" [deps.IntervalArithmetic.weakdeps] + Arblib = "fb37089c-8514-4489-9461-98f9c8763369" DiffRules = "b552c78f-8df3-52c6-915a-8e097449b14b" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.IntervalSets]] -git-tree-sha1 = "dba9ddf07f77f60450fe5d2e2beb9854d9a49bd0" +git-tree-sha1 = "5fbb102dcb8b1a858111ae81d56682376130517d" uuid = "8197267c-284f-5f27-9208-e0e47529a953" -version = "0.7.10" +version = "0.7.11" weakdeps = ["Random", "RecipesBase", "Statistics"] [deps.IntervalSets.extensions] @@ -1044,9 +1091,9 @@ uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" version = "1.3.1" [[deps.IrrationalConstants]] -git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" +git-tree-sha1 = "e2222959fbc6c19554dc15174c81bf7bf3aa691c" uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" -version = "0.2.2" +version = "0.2.4" [[deps.Isoband]] deps = ["isoband_jll"] @@ -1065,22 +1112,26 @@ uuid = "82899510-4779-5014-852e-03e436cf321d" version = "1.0.0" [[deps.JLD2]] -deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] -git-tree-sha1 = "91d501cb908df6f134352ad73cde5efc50138279" +deps = ["ChunkCodecLibZlib", "ChunkCodecLibZstd", "FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "ScopedValues"] +git-tree-sha1 = "da2e9b4d1abbebdcca0aa68afa0aa272102baad7" uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.5.11" +version = "0.6.2" +weakdeps = ["UnPack"] + + [deps.JLD2.extensions] + UnPackExt = "UnPack" [[deps.JLFzf]] -deps = ["Pipe", "REPL", "Random", "fzf_jll"] -git-tree-sha1 = "71b48d857e86bf7a1838c4736545699974ce79a2" +deps = ["REPL", "Random", "fzf_jll"] +git-tree-sha1 = "82f7acdc599b65e0f8ccd270ffa1467c21cb647b" uuid = "1019f520-868f-41f5-a6de-eb00f4b6a39c" -version = "0.1.9" +version = "0.1.11" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] @@ -1090,9 +1141,9 @@ version = "0.21.4" [[deps.JSON3]] deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] -git-tree-sha1 = "1d322381ef7b087548321d3f878cb4c9bd8f8f9b" +git-tree-sha1 = "411eccfe8aba0814ffa0fdf4860913ed09c34975" uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" -version = "1.14.1" +version = "1.14.3" [deps.JSON3.extensions] JSON3ArrowExt = ["ArrowTypes"] @@ -1102,15 +1153,15 @@ version = "1.14.1" [[deps.JpegTurbo]] deps = ["CEnum", "FileIO", "ImageCore", "JpegTurbo_jll", "TOML"] -git-tree-sha1 = "fa6d0bcff8583bac20f1ffa708c3913ca605c611" +git-tree-sha1 = "9496de8fb52c224a2e3f9ff403947674517317d9" uuid = "b835a17e-a41a-41e7-81f0-2f016b05efe0" -version = "0.1.5" +version = "0.1.6" [[deps.JpegTurbo_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "eac1206917768cb54957c65a615460d87b455fc1" +git-tree-sha1 = "4255f0032eafd6451d707a51d5f0248b8a165e4d" uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" -version = "3.1.1+0" +version = "3.1.3+0" [[deps.JuliaFormatter]] deps = ["CSTParser", "CommonMark", "DataStructures", "Glob", "PrecompileTools", "TOML", "Tokenize"] @@ -1120,33 +1171,42 @@ version = "1.0.62" [[deps.JuliaInterpreter]] deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] -git-tree-sha1 = "10da5154188682e5c0726823c2b5125957ec3778" +git-tree-sha1 = "d8337622fe53c05d16f031df24daf0270e53bc64" uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" -version = "0.9.38" +version = "0.10.5" [[deps.KernelDensity]] deps = ["Distributions", "DocStringExtensions", "FFTW", "Interpolations", "StatsBase"] -git-tree-sha1 = "7d703202e65efa1369de1279c162b915e245eed1" +git-tree-sha1 = "ba51324b894edaf1df3ab16e2cc6bc3280a2f1a7" uuid = "5ab0869b-81aa-558d-bb23-cbf5423bbe9b" -version = "0.6.9" +version = "0.6.10" [[deps.LAME_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "170b660facf5df5de098d866564877e119141cbd" +git-tree-sha1 = "059aabebaa7c82ccb853dd4a0ee9d17796f7e1bc" uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" -version = "3.100.2+0" +version = "3.100.3+0" [[deps.LERC_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "bf36f528eec6634efc60d7ec062008f171071434" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "aaafe88dccbd957a8d82f7d05be9b69172e0cee3" uuid = "88015f11-f218-50d7-93a8-a6af411a945d" -version = "3.0.0+1" +version = "4.0.1+0" [[deps.LLVMOpenMP_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "78211fb6cbc872f77cad3fc0b6cf647d923f4929" +git-tree-sha1 = "eb62a3deb62fc6d8822c0c4bef73e4412419c5d8" uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" -version = "18.1.7+0" +version = "18.1.8+0" + +[[deps.LRUCache]] +git-tree-sha1 = "5519b95a490ff5fe629c4a7aa3b3dfc9160498b3" +uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" +version = "1.6.2" +weakdeps = ["Serialization"] + + [deps.LRUCache.extensions] + SerializationExt = ["Serialization"] [[deps.LZO_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -1160,20 +1220,22 @@ uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" version = "1.4.0" [[deps.Latexify]] -deps = ["Format", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "OrderedCollections", "Requires"] -git-tree-sha1 = "ce5f5621cac23a86011836badfedf664a612cee4" +deps = ["Format", "Ghostscript_jll", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "OrderedCollections", "Requires"] +git-tree-sha1 = "44f93c47f9cd6c7e431f2f2091fcba8f01cd7e8f" uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" -version = "0.16.5" +version = "0.16.10" [deps.Latexify.extensions] DataFramesExt = "DataFrames" SparseArraysExt = "SparseArrays" SymEngineExt = "SymEngine" + TectonicExt = "tectonic_jll" [deps.Latexify.weakdeps] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SymEngine = "123dc426-2d89-5057-bbad-38513e3affd8" + tectonic_jll = "d7dd28d6-a5e6-559c-9131-7eb760cdacc5" [[deps.LayoutPointers]] deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface"] @@ -1226,34 +1288,22 @@ uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" version = "1.11.0" [[deps.Libepoxy_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libX11_jll"] -git-tree-sha1 = "7a0158b71f8be5c771e7a273183b2d0ac35278c5" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Xorg_libX11_jll"] +git-tree-sha1 = "94d068f57b4241dd090693b6aba63416892298de" uuid = "42c93a91-0102-5b3f-8f9d-e41de60ac950" -version = "1.5.10+0" +version = "1.5.11+0" [[deps.Libffi_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "27ecae93dd25ee0909666e6835051dd684cc035e" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "c8da7e6a91781c41a863611c7e966098d783c57a" uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" -version = "3.2.2+2" - -[[deps.Libgcrypt_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll"] -git-tree-sha1 = "8be878062e0ffa2c3f67bb58a595375eda5de80b" -uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" -version = "1.11.0+0" +version = "3.4.7+0" [[deps.Libglvnd_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll", "Xorg_libXext_jll"] -git-tree-sha1 = "ff3b4b9d35de638936a525ecd36e86a8bb919d11" +git-tree-sha1 = "d36c21b9e7c172a44a10484125024495e2625ac0" uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29" -version = "1.7.0+0" - -[[deps.Libgpg_error_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "df37206100d39f79b3376afb6b9cee4970041c61" -uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" -version = "1.51.1+0" +version = "1.7.1+1" [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -1263,21 +1313,21 @@ version = "1.18.0+0" [[deps.Libmount_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "89211ea35d9df5831fca5d33552c02bd33878419" +git-tree-sha1 = "3acf07f130a76f87c041cfb2ff7d7284ca67b072" uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" -version = "2.40.3+0" +version = "2.41.2+0" [[deps.Libtiff_jll]] -deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "LERC_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"] -git-tree-sha1 = "3eb79b0ca5764d4799c06699573fd8f533259713" +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "LERC_jll", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "f04133fe05eff1667d2054c53d59f9122383fe05" uuid = "89763e89-9b03-5906-acba-b20f662cd828" -version = "4.4.0+0" +version = "4.7.2+0" [[deps.Libuuid_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "e888ad02ce716b319e6bdb985d2ef300e7089889" +git-tree-sha1 = "2a7a12fc0a4e7fb773450d17975322aa77142106" uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" -version = "2.40.3+0" +version = "2.41.2+0" [[deps.Libz]] deps = ["BufferedStreams", "Random", "Test"] @@ -1291,10 +1341,10 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" version = "1.11.0" [[deps.LittleCMS_jll]] -deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pkg"] -git-tree-sha1 = "110897e7db2d6836be22c18bffd9422218ee6284" +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll"] +git-tree-sha1 = "8e6a74641caf3b84800f2ccd55dc7ab83893c10b" uuid = "d3a379c0-f9a3-5b72-a4c0-6bf4d2e8af0f" -version = "2.12.0+0" +version = "2.17.0+0" [[deps.LogExpFunctions]] deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] @@ -1324,9 +1374,9 @@ version = "1.1.0" [[deps.LoopVectorization]] deps = ["ArrayInterface", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] -git-tree-sha1 = "8084c25a250e00ae427a379a5b607e7aed96a2dd" +git-tree-sha1 = "e5afce7eaf5b5ca0d444bcb4dc4fd78c54cbbac0" uuid = "bdcacae8-1622-11e9-2a5c-532679323890" -version = "0.12.171" +version = "0.12.172" [deps.LoopVectorization.extensions] ForwardDiffExt = ["ChainRulesCore", "ForwardDiff"] @@ -1338,10 +1388,10 @@ version = "0.12.171" SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" [[deps.LoweredCodeUtils]] -deps = ["JuliaInterpreter"] -git-tree-sha1 = "688d6d9e098109051ae33d126fcfc88c4ce4a021" +deps = ["CodeTracking", "Compiler", "JuliaInterpreter"] +git-tree-sha1 = "73b98709ad811a6f81d84e105f4f695c229385ba" uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" -version = "3.1.0" +version = "3.4.3" [[deps.MIMEs]] git-tree-sha1 = "65f28ad4b594aebe22157d6fac869786a255b7eb" @@ -1350,14 +1400,14 @@ version = "0.1.4" [[deps.MKL_jll]] deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] -git-tree-sha1 = "ed4097130e3dd3721814b5f277da72f48905e80c" +git-tree-sha1 = "282cadc186e7b2ae0eeadbd7a4dffed4196ae2aa" uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" -version = "2025.0.1+0" +version = "2025.2.0+0" [[deps.MacroTools]] -git-tree-sha1 = "72aebe0b5051e5143a079a4685a46da330a40472" +git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.15" +version = "0.5.16" [[deps.Makie]] deps = ["Animations", "Base64", "CRC32c", "ColorBrewer", "ColorSchemes", "ColorTypes", "Colors", "Contour", "Dates", "DelaunayTriangulation", "Distributions", "DocStringExtensions", "Downloads", "FFMPEG_jll", "FileIO", "FilePaths", "FixedPointNumbers", "Format", "FreeType", "FreeTypeAbstraction", "GeometryBasics", "GridLayoutBase", "ImageBase", "ImageIO", "InteractiveUtils", "Interpolations", "IntervalSets", "InverseFunctions", "Isoband", "KernelDensity", "LaTeXStrings", "LinearAlgebra", "MacroTools", "MakieCore", "Markdown", "MathTeXEngine", "Observables", "OffsetArrays", "PNGFiles", "Packing", "PlotUtils", "PolygonOps", "PrecompileTools", "Printf", "REPL", "Random", "RelocatableFolders", "Scratch", "ShaderAbstractions", "Showoff", "SignedDistanceFields", "SparseArrays", "Statistics", "StatsBase", "StatsFuns", "StructArrays", "TriplotBase", "UnicodeFun", "Unitful"] @@ -1373,9 +1423,9 @@ version = "0.9.0" [[deps.Malt]] deps = ["Distributed", "Logging", "RelocatableFolders", "Serialization", "Sockets"] -git-tree-sha1 = "02a728ada9d6caae583a0f87c1dd3844f99ec3fd" +git-tree-sha1 = "636abf4fb184be05888dacc0b636fe0911d6d746" uuid = "36869731-bdee-424d-aa32-cab38c994e3b" -version = "1.1.2" +version = "1.2.0" [[deps.ManualMemory]] git-tree-sha1 = "bcaef4fc7a0cfe2cba636d84cda54b5e4e4ca3cd" @@ -1394,9 +1444,9 @@ version = "1.11.0" [[deps.MathTeXEngine]] deps = ["AbstractTrees", "Automa", "DataStructures", "FreeTypeAbstraction", "GeometryBasics", "LaTeXStrings", "REPL", "RelocatableFolders", "UnicodeFun"] -git-tree-sha1 = "f45c8916e8385976e1ccd055c9874560c257ab13" +git-tree-sha1 = "a370fef694c109e1950836176ed0d5eabbb65479" uuid = "0a4f8689-d25c-4efe-a92b-7142dfc1aa53" -version = "0.6.2" +version = "0.6.6" [[deps.MbedTLS]] deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] @@ -1416,9 +1466,9 @@ version = "0.3.2" [[deps.MetaGraphs]] deps = ["Graphs", "JLD2", "Random"] -git-tree-sha1 = "e9650bea7f91c3397eb9ae6377343963a22bf5b8" +git-tree-sha1 = "3a8f462a180a9d735e340f4e8d5f364d411da3a4" uuid = "626554b9-1ddb-594c-aa3c-2596fe9399a5" -version = "0.8.0" +version = "0.8.1" [[deps.Millboard]] git-tree-sha1 = "ea6a5b7e56e76d8051023faaa11d91d1d881dac3" @@ -1459,9 +1509,9 @@ version = "1.2.1" [[deps.NaNMath]] deps = ["OpenLibm_jll"] -git-tree-sha1 = "dbad50672562ce422a406d172fb19fc1f0166f06" +git-tree-sha1 = "9b8215b1ee9e78a293f99797cd31375471b2bcae" uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" -version = "1.1.0" +version = "1.1.3" [[deps.NativeFileDialog]] deps = ["FilePathsBase", "NativeFileDialog_jll"] @@ -1482,9 +1532,9 @@ version = "1.0.0" [[deps.NearestNeighbors]] deps = ["Distances", "StaticArrays"] -git-tree-sha1 = "8a3271d8309285f4db73b4f662b1b290c715e85e" +git-tree-sha1 = "ca7e18198a166a1f3eb92a3650d53d94ed8ca8a1" uuid = "b8a86587-4115-5ab1-83bc-aa920d37bbce" -version = "0.4.21" +version = "0.4.22" [[deps.Netpbm]] deps = ["FileIO", "ImageCore", "ImageMetadata"] @@ -1519,19 +1569,25 @@ uuid = "510215fc-4207-5dde-b226-833fc4488ee2" version = "0.5.5" [[deps.OffsetArrays]] -git-tree-sha1 = "5e1897147d1ff8d98883cda2be2187dcf57d8f0c" +git-tree-sha1 = "117432e406b5c023f665fa73dc26e79ec3630151" uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -version = "1.15.0" +version = "1.17.0" weakdeps = ["Adapt"] [deps.OffsetArrays.extensions] OffsetArraysAdaptExt = "Adapt" [[deps.Ogg_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "887579a3eb005446d514ab7aeac5d1d027658b8f" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b6aa4566bb7ae78498a5e68943863fa8b5231b59" uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" -version = "1.3.5+1" +version = "1.3.6+0" + +[[deps.OpenBLASConsistentFPCSR_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "567515ca155d0020a45b05175449b499c63e7015" +uuid = "6cdc7f73-28fd-5e50-80fb-958a8875b1af" +version = "0.3.29+0" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] @@ -1551,33 +1607,33 @@ uuid = "18a262bb-aa17-5467-a713-aee519bc75cb" version = "3.2.4+0" [[deps.OpenJpeg_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libtiff_jll", "LittleCMS_jll", "Pkg", "libpng_jll"] -git-tree-sha1 = "76374b6e7f632c130e78100b166e5a48464256f8" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libtiff_jll", "LittleCMS_jll", "libpng_jll"] +git-tree-sha1 = "7dc7028a10d1408e9103c0a77da19fdedce4de6c" uuid = "643b3616-a352-519d-856d-80112ee9badc" -version = "2.4.0+0" +version = "2.5.4+0" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" -version = "0.8.1+2" +version = "0.8.5+0" [[deps.OpenSSH_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "1b2f042897343a9dfdcc9366e4ecbd3d00780c49" +deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] +git-tree-sha1 = "cf28501224f22b2b4d237d3cc7215c7c0cb52f6a" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "8.9.0+1" +version = "9.9.2+0" [[deps.OpenSSL]] deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] -git-tree-sha1 = "38cb508d080d21dc1128f7fb04f20387ed4c0af4" +git-tree-sha1 = "f1a7e086c677df53e064e0fdd2c9d0b0833e3f6e" uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" -version = "1.4.3" +version = "1.5.0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "ad31332567b189f508a3ea8957a2640b1147ab00" +git-tree-sha1 = "2ae7d4ddec2e13ad3bddf5c0796f7547cf682391" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "1.1.23+1" +version = "3.5.2+0" [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] @@ -1587,14 +1643,14 @@ version = "0.5.6+0" [[deps.Opus_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6703a85cb3781bd5909d48730a67205f3f31a575" +git-tree-sha1 = "c392fc5dd032381919e3b22dd32d6443760ce7ea" uuid = "91d4177d-7536-5919-b921-800302f37372" -version = "1.3.3+0" +version = "1.5.2+0" [[deps.OrderedCollections]] -git-tree-sha1 = "12f1439c4f986bb868acda6ea33ebc78e19b95ad" +git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.7.0" +version = "1.8.1" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] @@ -1603,15 +1659,15 @@ version = "10.42.0+1" [[deps.PDMats]] deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "966b85253e959ea89c53a9abebbf2e964fbf593b" +git-tree-sha1 = "f07c06228a1c670ae4c87d1276b92c7c597fdda0" uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" -version = "0.11.32" +version = "0.11.35" [[deps.PNGFiles]] deps = ["Base64", "CEnum", "ImageCore", "IndirectArrays", "OffsetArrays", "libpng_jll"] -git-tree-sha1 = "67186a2bc9a90f9f85ff3cc8277868961fb57cbd" +git-tree-sha1 = "cf181f0b1e6a18dfeb0ee8acc4a9d1672499626c" uuid = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883" -version = "0.4.3" +version = "0.4.4" [[deps.Packing]] deps = ["GeometryBasics"] @@ -1627,9 +1683,9 @@ version = "0.5.12" [[deps.Pango_jll]] deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl"] -git-tree-sha1 = "ed6834e95bd326c52d5675b4181386dfbe885afb" +git-tree-sha1 = "1f7f9bbd5f7a2e5a9f7d96e51c9754454ea7f60b" uuid = "36c8627f-9965-5494-a995-c6b170f724f3" -version = "1.55.5+0" +version = "1.56.4+0" [[deps.Parameters]] deps = ["OrderedCollections", "UnPack"] @@ -1639,20 +1695,15 @@ version = "0.12.3" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.1" - -[[deps.Pipe]] -git-tree-sha1 = "6842804e7867b115ca9de748a0cf6b364523c16d" -uuid = "b98c9c47-44ae-5843-9183-064241ee97a0" -version = "1.3.0" +version = "2.8.3" [[deps.Pixman_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "Libdl"] -git-tree-sha1 = "35621f10a7531bc8fa58f74610b1bfb70a3cfc6b" +git-tree-sha1 = "db76b1ecd5e9715f3d043cec13b2ec93ce015d53" uuid = "30392449-352a-5448-841d-b1acce4e97dc" -version = "0.43.4+0" +version = "0.44.2+0" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] @@ -1688,10 +1739,10 @@ uuid = "a03496cd-edff-5a9b-9e67-9cda94a718b5" version = "0.8.19" [[deps.Plots]] -deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "JLFzf", "JSON", "LaTeXStrings", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "Pkg", "PlotThemes", "PlotUtils", "PrecompileTools", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "RelocatableFolders", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "TOML", "UUIDs", "UnicodeFun", "UnitfulLatexify", "Unzip"] -git-tree-sha1 = "f202a1ca4f6e165238d8175df63a7e26a51e04dc" +deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "JLFzf", "JSON", "LaTeXStrings", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "Pkg", "PlotThemes", "PlotUtils", "PrecompileTools", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "RelocatableFolders", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "TOML", "UUIDs", "UnicodeFun", "Unzip"] +git-tree-sha1 = "12ce661880f8e309569074a61d3767e5756a199f" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -version = "1.40.7" +version = "1.41.1" [deps.Plots.extensions] FileIOExt = "FileIO" @@ -1708,16 +1759,16 @@ version = "1.40.7" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" [[deps.Pluto]] -deps = ["Base64", "Configurations", "Dates", "Downloads", "ExpressionExplorer", "FileWatching", "FuzzyCompletions", "HTTP", "HypertextLiteral", "InteractiveUtils", "Logging", "LoggingExtras", "MIMEs", "Malt", "Markdown", "MsgPack", "Pkg", "PlutoDependencyExplorer", "PrecompileSignatures", "PrecompileTools", "REPL", "RegistryInstances", "RelocatableFolders", "Scratch", "Sockets", "TOML", "Tables", "URIs", "UUIDs"] -git-tree-sha1 = "b5509a2e4d4c189da505b780e3f447d1e38a0350" +deps = ["Base64", "Configurations", "Dates", "Downloads", "ExpressionExplorer", "FileWatching", "GracefulPkg", "HTTP", "HypertextLiteral", "InteractiveUtils", "LRUCache", "Logging", "LoggingExtras", "MIMEs", "Malt", "Markdown", "MsgPack", "Pkg", "PlutoDependencyExplorer", "PrecompileSignatures", "PrecompileTools", "REPL", "RegistryInstances", "RelocatableFolders", "Scratch", "Sockets", "TOML", "Tables", "URIs", "UUIDs"] +git-tree-sha1 = "64ff678257a6e59364ac2c094642348104c17443" uuid = "c3e4b0f8-55cb-11ea-2926-15256bba5781" -version = "0.20.4" +version = "0.20.18" [[deps.PlutoDependencyExplorer]] deps = ["ExpressionExplorer", "InteractiveUtils", "Markdown"] -git-tree-sha1 = "e0864c15334d2c4bac8137ce3359f1174565e719" +git-tree-sha1 = "c3e5073a977b1c58b2d55c1ec187c3737e64e6af" uuid = "72656b73-756c-7461-726b-72656b6b696b" -version = "1.2.0" +version = "1.2.2" [[deps.PolyesterWeave]] deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"] @@ -1732,9 +1783,9 @@ version = "0.1.2" [[deps.Polynomials]] deps = ["LinearAlgebra", "OrderedCollections", "RecipesBase", "Requires", "Setfield", "SparseArrays"] -git-tree-sha1 = "27f6107dc202e2499f0750c628a848ce5d6e77f5" +git-tree-sha1 = "2660ba4e0e9354cf0f3deeefb61531bbdac5495d" uuid = "f27b6e38-b328-58d1-80ce-0feddd5e7a45" -version = "4.0.13" +version = "4.0.21" [deps.Polynomials.extensions] PolynomialsChainRulesCoreExt = "ChainRulesCore" @@ -1767,9 +1818,9 @@ version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.PrettyTables]] deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] @@ -1784,9 +1835,9 @@ version = "1.11.0" [[deps.ProgressMeter]] deps = ["Distributed", "Printf"] -git-tree-sha1 = "8f6bc219586aef8baf0ff9a5fe16ee9c70cb65e4" +git-tree-sha1 = "fbb92c6c56b34e1a2c4c36058f68f332bec840e7" uuid = "92933f4c-e287-5a05-a399-4b506db050ca" -version = "1.10.2" +version = "1.11.0" [[deps.PtrArrays]] git-tree-sha1 = "1d36ef11a9aaf1e8b74dacc6a731dd1de8fd493d" @@ -1801,9 +1852,9 @@ version = "1.96.4" [[deps.PyPlot]] deps = ["Colors", "LaTeXStrings", "PyCall", "Sockets", "Test", "VersionParsing"] -git-tree-sha1 = "0371ca706e3f295481cbf94c8c36692b072285c2" +git-tree-sha1 = "d2c2b8627bbada1ba00af2951946fb8ce6012c05" uuid = "d330b81b-6aea-500a-939a-2ce795aea3ee" -version = "2.11.5" +version = "2.11.6" [[deps.QOI]] deps = ["ColorTypes", "FileIO", "FixedPointNumbers"] @@ -1811,17 +1862,35 @@ git-tree-sha1 = "8b3fc30bc0390abdce15f8822c889f669baed73d" uuid = "4b34888f-f399-49d4-9bb3-47ed5cae4e65" version = "1.0.1" -[[deps.Qt5Base_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"] -git-tree-sha1 = "0c03844e2231e12fda4d0086fd7cbe4098ee8dc5" -uuid = "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1" -version = "5.15.3+2" +[[deps.Qt6Base_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Vulkan_Loader_jll", "Xorg_libSM_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_cursor_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "libinput_jll", "xkbcommon_jll"] +git-tree-sha1 = "eb38d376097f47316fe089fc62cb7c6d85383a52" +uuid = "c0090381-4147-56d7-9ebc-da0b1113ec56" +version = "6.8.2+1" + +[[deps.Qt6Declarative_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Qt6Base_jll", "Qt6ShaderTools_jll"] +git-tree-sha1 = "da7adf145cce0d44e892626e647f9dcbe9cb3e10" +uuid = "629bc702-f1f5-5709-abd5-49b8460ea067" +version = "6.8.2+1" + +[[deps.Qt6ShaderTools_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Qt6Base_jll"] +git-tree-sha1 = "9eca9fc3fe515d619ce004c83c31ffd3f85c7ccf" +uuid = "ce943373-25bb-56aa-8eca-768745ed7b5a" +version = "6.8.2+1" + +[[deps.Qt6Wayland_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Qt6Base_jll", "Qt6Declarative_jll"] +git-tree-sha1 = "e1d5e16d0f65762396f9ca4644a5f4ddab8d452b" +uuid = "e99dba38-086e-5de3-a5b1-6e4c66e897c3" +version = "6.8.2+1" [[deps.QuadGK]] deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "cda3b045cf9ef07a08ad46731f5a3165e56cf3da" +git-tree-sha1 = "9da16da70037ba9d701192e27befedefb91ec284" uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.11.1" +version = "2.11.2" [deps.QuadGK.extensions] QuadGKEnzymeExt = "Enzyme" @@ -1902,10 +1971,10 @@ uuid = "05181044-ff0b-4ac5-8273-598c1e38db00" version = "1.0.1" [[deps.RemoteREPL]] -deps = ["Logging", "OpenSSH_jll", "REPL", "ReplMaker", "Serialization", "Sockets"] -git-tree-sha1 = "42618f51c9c813f0c665dc4fe56dc2d5d90baab9" +deps = ["Logging", "OpenSSH_jll", "REPL", "ReplMaker", "Serialization", "Sockets", "UUIDs"] +git-tree-sha1 = "1d75634838e7a07872c3339aed8b86bfce9afafc" uuid = "1bd9f7bb-701c-4338-bec7-ac987af7c555" -version = "0.2.17" +version = "0.3.0" [[deps.ReplMaker]] deps = ["REPL", "Unicode"] @@ -1915,15 +1984,15 @@ version = "0.2.7" [[deps.Requires]] deps = ["UUIDs"] -git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" uuid = "ae029012-a4dd-5104-9daa-d747884805df" -version = "1.3.0" +version = "1.3.1" [[deps.Revise]] deps = ["CodeTracking", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "REPL", "Requires", "UUIDs", "Unicode"] -git-tree-sha1 = "9bb80533cb9769933954ea4ffbecb3025a783198" +git-tree-sha1 = "d852eba0cc08181083a58d5eb9dccaec3129cb03" uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" -version = "3.7.2" +version = "3.9.0" weakdeps = ["Distributed"] [deps.Revise.extensions] @@ -1977,11 +2046,22 @@ git-tree-sha1 = "456f610ca2fbd1c14f5fcf31c6bfadc55e7d66e0" uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa" version = "0.6.43" +[[deps.SciMLPublic]] +git-tree-sha1 = "ed647f161e8b3f2973f24979ec074e8d084f1bee" +uuid = "431bcebd-1456-4ced-9d72-93c2757fff0b" +version = "1.0.0" + +[[deps.ScopedValues]] +deps = ["HashArrayMappedTries", "Logging"] +git-tree-sha1 = "c3b2323466378a2ba15bea4b2f73b081e022f473" +uuid = "7e506255-f358-4e82-b7e4-beb19740aa63" +version = "1.5.0" + [[deps.Scratch]] deps = ["Dates"] -git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" +git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109" uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.2.1" +version = "1.3.0" [[deps.SentinelArrays]] deps = ["Dates", "Random"] @@ -1995,9 +2075,9 @@ version = "1.11.0" [[deps.Setfield]] deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] -git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac" +git-tree-sha1 = "c5391c6ace3bc430ca630251d02ea9687169ca68" uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" -version = "1.1.1" +version = "1.1.2" [[deps.ShaderAbstractions]] deps = ["ColorTypes", "FixedPointNumbers", "GeometryBasics", "LinearAlgebra", "Observables", "StaticArrays"] @@ -2029,21 +2109,21 @@ version = "1.2.0" [[deps.SimpleTraits]] deps = ["InteractiveUtils", "MacroTools"] -git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" +git-tree-sha1 = "be8eeac05ec97d379347584fa9fe2f5f76795bcb" uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" -version = "0.9.4" +version = "0.9.5" [[deps.SimpleWeightedGraphs]] deps = ["Graphs", "LinearAlgebra", "Markdown", "SparseArrays"] -git-tree-sha1 = "4b33e0e081a825dbfaf314decf58fa47e53d6acb" +git-tree-sha1 = "3e5f165e58b18204aed03158664c4982d691f454" uuid = "47aef6b3-ad0c-573a-a1e2-d07658019622" -version = "1.4.0" +version = "1.5.0" [[deps.Sixel]] deps = ["Dates", "FileIO", "ImageCore", "IndirectArrays", "OffsetArrays", "REPL", "libsixel_jll"] -git-tree-sha1 = "2da10356e31327c7096832eb9cd86307a50b1eb6" +git-tree-sha1 = "0494aed9501e7fb65daba895fb7fd57cc38bc743" uuid = "45858cf5-a6b0-47a3-bbea-62219f50df47" -version = "0.1.3" +version = "0.1.5" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" @@ -2051,9 +2131,9 @@ version = "1.11.0" [[deps.SortingAlgorithms]] deps = ["DataStructures"] -git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" +git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9" uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.2.1" +version = "1.2.2" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] @@ -2062,9 +2142,9 @@ version = "1.11.0" [[deps.SpecialFunctions]] deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] -git-tree-sha1 = "64cca0c26b4f31ba18f13f6c12af7c85f478cfde" +git-tree-sha1 = "41852b8679f78c8d8961eeadc8f62cef861a52e3" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "2.5.0" +version = "2.5.1" weakdeps = ["ChainRulesCore"] [deps.SpecialFunctions.extensions] @@ -2072,21 +2152,21 @@ weakdeps = ["ChainRulesCore"] [[deps.StableRNGs]] deps = ["Random"] -git-tree-sha1 = "83e6cce8324d49dfaf9ef059227f91ed4441a8e5" +git-tree-sha1 = "95af145932c2ed859b63329952ce8d633719f091" uuid = "860ef19b-820b-49d6-a774-d7a799459cd3" -version = "1.0.2" +version = "1.0.3" [[deps.StackViews]] deps = ["OffsetArrays"] -git-tree-sha1 = "46e589465204cd0c08b4bd97385e4fa79a0c770c" +git-tree-sha1 = "be1cf4eb0ac528d96f5115b4ed80c26a8d8ae621" uuid = "cae243ae-269e-4f55-b966-ac2d0dc13c15" -version = "0.1.1" +version = "0.1.2" [[deps.Static]] -deps = ["CommonWorldInvalidations", "IfElse", "PrecompileTools"] -git-tree-sha1 = "87d51a3ee9a4b0d2fe054bdd3fc2436258db2603" +deps = ["CommonWorldInvalidations", "IfElse", "PrecompileTools", "SciMLPublic"] +git-tree-sha1 = "1e44e7b1dbb5249876d84c32466f8988a6b41bbb" uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3" -version = "1.1.1" +version = "1.3.0" [[deps.StaticArrayInterface]] deps = ["ArrayInterface", "Compat", "IfElse", "LinearAlgebra", "PrecompileTools", "Static"] @@ -2101,9 +2181,9 @@ weakdeps = ["OffsetArrays", "StaticArrays"] [[deps.StaticArrays]] deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] -git-tree-sha1 = "47091a0340a675c738b1304b58161f3b0839d454" +git-tree-sha1 = "b8693004b385c842357406e3af647701fe783f98" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.9.10" +version = "1.9.15" weakdeps = ["ChainRulesCore", "Statistics"] [deps.StaticArrays.extensions] @@ -2127,21 +2207,21 @@ weakdeps = ["SparseArrays"] [[deps.StatsAPI]] deps = ["LinearAlgebra"] -git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" +git-tree-sha1 = "9d72a13a3f4dd3795a195ac5a44d7d6ff5f552ff" uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" -version = "1.7.0" +version = "1.7.1" [[deps.StatsBase]] deps = ["AliasTables", "DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] -git-tree-sha1 = "29321314c920c26684834965ec2ce0dacc9cf8e5" +git-tree-sha1 = "2c962245732371acd51700dbb268af311bddd719" uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.34.4" +version = "0.34.6" [[deps.StatsFuns]] deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] -git-tree-sha1 = "b423576adc27097764a90e163157bcfc9acf0f46" +git-tree-sha1 = "8e45cecc66f3b42633b8ce14d431e8e57a3e242e" uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" -version = "1.3.2" +version = "1.5.0" weakdeps = ["ChainRulesCore", "InverseFunctions"] [deps.StatsFuns.extensions] @@ -2150,9 +2230,9 @@ weakdeps = ["ChainRulesCore", "InverseFunctions"] [[deps.Stipple]] deps = ["Dates", "FilePathsBase", "Genie", "GenieSession", "GenieSessionFileSession", "JSON3", "Logging", "MacroTools", "Mixers", "Observables", "OrderedCollections", "Parameters", "Pkg", "PrecompileTools", "Random", "Reexport", "Requires", "StructTypes", "Tables"] -git-tree-sha1 = "0a32740b08234a7876cdfdf1714673d64c9dd148" +git-tree-sha1 = "9455fc54d50b4a6e6cb986eda7e8a0d09938ba06" uuid = "4acbeb90-81a0-11ea-1966-bdaff8155998" -version = "0.28.22" +version = "0.28.23" weakdeps = ["DataFrames", "JSON", "OffsetArrays"] [deps.Stipple.extensions] @@ -2162,9 +2242,9 @@ weakdeps = ["DataFrames", "JSON", "OffsetArrays"] [[deps.StipplePivotTable]] deps = ["Stipple", "StippleUI", "Tables"] -git-tree-sha1 = "02c29d8464f1c26f659d1aa94d69a986ecaafea4" +git-tree-sha1 = "77bb654bf22c50872878ccf4653949d95808a055" uuid = "adab0226-ee3c-44de-8f2f-393b1bf1e936" -version = "0.3.1" +version = "0.3.2" [[deps.StipplePlotly]] deps = ["DataFrames", "Genie", "Requires", "Stipple"] @@ -2178,21 +2258,21 @@ weakdeps = ["PlotlyBase"] [[deps.StippleTable]] deps = ["Stipple", "StippleUI"] -git-tree-sha1 = "c3a06836672a19fafdb8c600acc077746a4200b2" +git-tree-sha1 = "f4a36ed4b56181d459b47bd16429308a03e05c1c" uuid = "5bb4b094-0818-496d-ae94-fd777e10e8ba" -version = "1.0.1" +version = "1.0.3" [[deps.StippleTabs]] deps = ["Stipple", "StippleUI"] -git-tree-sha1 = "f8721b7491bfd239c69eace4c4acb0e74962c28f" +git-tree-sha1 = "cf1440b0d7510c98553ed75c7ba0842780a45319" uuid = "29f11006-3202-4091-a17b-0a26282d76dd" -version = "1.0.0" +version = "1.1.0" [[deps.StippleUI]] deps = ["Colors", "Dates", "Genie", "OrderedCollections", "PrecompileTools", "Stipple", "Tables"] -git-tree-sha1 = "099e2678af62149317b0ae69874dd617924182bc" +git-tree-sha1 = "fb442128bab27a90c9356f9e5ed39dada2b3d207" uuid = "a3c5d34a-b254-4859-a8fa-b86abb7e84a3" -version = "0.23.5" +version = "0.23.7" weakdeps = ["DataFrames"] [deps.StippleUI.extensions] @@ -2206,15 +2286,15 @@ version = "0.3.7" [[deps.StringManipulation]] deps = ["PrecompileTools"] -git-tree-sha1 = "a6b1675a536c5ad1a60e5a5153e1fee12eb146e3" +git-tree-sha1 = "725421ae8e530ec29bcbdddbe91ff8053421d023" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.4.0" +version = "0.4.1" [[deps.StructArrays]] deps = ["ConstructionBase", "DataAPI", "Tables"] -git-tree-sha1 = "5a3a31c41e15a1e042d60f2f4942adccba05d3c9" +git-tree-sha1 = "8ad2e38cbb812e29348719cc63580ec1dfeb9de4" uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" -version = "0.7.0" +version = "0.7.1" [deps.StructArrays.extensions] StructArraysAdaptExt = "Adapt" @@ -2263,9 +2343,9 @@ version = "1.0.1" [[deps.Tables]] deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] -git-tree-sha1 = "598cd7c1f68d1e205689b1c2fe65a9f85846f297" +git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.12.0" +version = "1.12.1" [[deps.Tar]] deps = ["ArgTools", "SHA"] @@ -2290,15 +2370,15 @@ version = "1.0.2" [[deps.ThreadingUtilities]] deps = ["ManualMemory"] -git-tree-sha1 = "eda08f7e9818eb53661b3deb74e3159460dfbc27" +git-tree-sha1 = "d969183d3d244b6c33796b5ed01ab97328f2db85" uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5" -version = "0.5.2" +version = "0.5.5" [[deps.TiffImages]] -deps = ["ColorTypes", "DataStructures", "DocStringExtensions", "FileIO", "FixedPointNumbers", "IndirectArrays", "Inflate", "Mmap", "OffsetArrays", "PkgVersion", "ProgressMeter", "SIMD", "UUIDs"] -git-tree-sha1 = "3c0faa42f2bd3c6d994b06286bba2328eae34027" +deps = ["ColorTypes", "DataStructures", "DocStringExtensions", "FileIO", "FixedPointNumbers", "IndirectArrays", "Inflate", "Mmap", "OffsetArrays", "PkgVersion", "PrecompileTools", "ProgressMeter", "SIMD", "UUIDs"] +git-tree-sha1 = "98b9352a24cb6a2066f9ababcc6802de9aed8ad8" uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69" -version = "0.11.2" +version = "0.11.6" [[deps.TiledIteration]] deps = ["OffsetArrays", "StaticArrayInterface"] @@ -2317,9 +2397,9 @@ uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" version = "0.11.3" [[deps.Tricks]] -git-tree-sha1 = "6cae795a5a9313bbb4f60683f7263318fc7d1505" +git-tree-sha1 = "372b90fe551c019541fafc6ff034199dc19c8436" uuid = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775" -version = "0.1.10" +version = "0.1.12" [[deps.TriplotBase]] git-tree-sha1 = "4d4ed7f294cda19382ff7de4c137d24d16adc89b" @@ -2333,9 +2413,9 @@ uuid = "30578b45-9adc-5946-b283-645ec420af67" version = "0.4.1" [[deps.URIs]] -git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" +git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -version = "1.5.1" +version = "1.6.1" [[deps.UUIDs]] deps = ["Random", "SHA"] @@ -2359,20 +2439,24 @@ version = "0.4.1" [[deps.Unitful]] deps = ["Dates", "LinearAlgebra", "Random"] -git-tree-sha1 = "c0667a8e676c53d390a09dc6870b3d8d6650e2bf" +git-tree-sha1 = "cec2df8cf14e0844a8c4d770d12347fda5931d72" uuid = "1986cc42-f94f-5a68-af5c-568840ba703d" -version = "1.22.0" -weakdeps = ["ConstructionBase", "InverseFunctions"] +version = "1.25.0" [deps.Unitful.extensions] ConstructionBaseUnitfulExt = "ConstructionBase" + ForwardDiffExt = "ForwardDiff" InverseFunctionsUnitfulExt = "InverseFunctions" + LatexifyExt = ["Latexify", "LaTeXStrings"] + PrintfExt = "Printf" -[[deps.UnitfulLatexify]] -deps = ["LaTeXStrings", "Latexify", "Unitful"] -git-tree-sha1 = "975c354fcd5f7e1ddcc1f1a23e6e091d99e99bc8" -uuid = "45397f5d-5981-4c77-b2b3-fc36d6e9b728" -version = "1.6.4" + [deps.Unitful.weakdeps] + ConstructionBase = "187b0558-2788-49d3-abe0-74a17ed4e7c9" + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" + Latexify = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" + Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" [[deps.Unzip]] git-tree-sha1 = "ca0969166a028236229f63514992fc073799bb78" @@ -2381,26 +2465,32 @@ version = "0.2.0" [[deps.VectorizationBase]] deps = ["ArrayInterface", "CPUSummary", "HostCPUFeatures", "IfElse", "LayoutPointers", "Libdl", "LinearAlgebra", "SIMDTypes", "Static", "StaticArrayInterface"] -git-tree-sha1 = "4ab62a49f1d8d9548a1c8d1a75e5f55cf196f64e" +git-tree-sha1 = "d1d9a935a26c475ebffd54e9c7ad11627c43ea85" uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f" -version = "0.21.71" +version = "0.21.72" [[deps.VersionParsing]] git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868" uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" version = "1.3.0" -[[deps.Wayland_jll]] -deps = ["Artifacts", "EpollShim_jll", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"] -git-tree-sha1 = "85c7811eddec9e7f22615371c3cc81a504c508ee" -uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89" -version = "1.21.0+2" +[[deps.Vulkan_Loader_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Wayland_jll", "Xorg_libX11_jll", "Xorg_libXrandr_jll", "xkbcommon_jll"] +git-tree-sha1 = "2f0486047a07670caad3a81a075d2e518acc5c59" +uuid = "a44049a8-05dd-5a78-86c9-5fde0876e88c" +version = "1.3.243+0" -[[deps.Wayland_protocols_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "5db3e9d307d32baba7067b13fc7b5aa6edd4a19a" -uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91" -version = "1.36.0+0" +[[deps.Wayland_jll]] +deps = ["Artifacts", "EpollShim_jll", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll"] +git-tree-sha1 = "96478df35bbc2f3e1e791bc7a3d0eeee559e60e9" +uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89" +version = "1.24.0+0" + +[[deps.WeakRefStrings]] +deps = ["DataAPI", "InlineStrings", "Parsers"] +git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" +version = "1.4.2" [[deps.WebP]] deps = ["CEnum", "ColorTypes", "FileIO", "FixedPointNumbers", "ImageCore", "libwebp_jll"] @@ -2414,167 +2504,184 @@ git-tree-sha1 = "c1a7aa6219628fcd757dede0ca95e245c5cd9511" uuid = "efce3f68-66dc-5838-9240-27a6d6f5f9b6" version = "1.0.0" +[[deps.WorkerUtilities]] +git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" +uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" +version = "1.6.1" + [[deps.XML2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] git-tree-sha1 = "04a51d15436a572301b5abbb9d099713327e9fc4" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" version = "2.10.4+0" -[[deps.XSLT_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "XML2_jll", "Zlib_jll"] -git-tree-sha1 = "7d1671acbe47ac88e981868a078bd6b4e27c5191" -uuid = "aed1982a-8fda-507f-9586-7b0439959a61" -version = "1.1.42+0" +[[deps.XZ_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "fee71455b0aaa3440dfdd54a9a36ccef829be7d4" +uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" +version = "5.8.1+0" + +[[deps.Xorg_libICE_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "a3ea76ee3f4facd7a64684f9af25310825ee3668" +uuid = "f67eecfb-183a-506d-b269-f58e52b52d7c" +version = "1.1.2+0" + +[[deps.Xorg_libSM_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libICE_jll"] +git-tree-sha1 = "9c7ad99c629a44f81e7799eb05ec2746abb5d588" +uuid = "c834827a-8449-5923-a945-d239c165b7dd" +version = "1.2.6+0" [[deps.Xorg_libX11_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] -git-tree-sha1 = "9dafcee1d24c4f024e7edc92603cedba72118283" +git-tree-sha1 = "b5899b25d17bf1889d25906fb9deed5da0c15b3b" uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" -version = "1.8.6+3" +version = "1.8.12+0" [[deps.Xorg_libXau_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "e9216fdcd8514b7072b43653874fd688e4c6c003" +git-tree-sha1 = "aa1261ebbac3ccc8d16558ae6799524c450ed16b" uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" -version = "1.0.12+0" +version = "1.0.13+0" [[deps.Xorg_libXcomposite_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll"] -git-tree-sha1 = "7c688ca9c957837539bbe1c53629bb871025e423" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXfixes_jll"] +git-tree-sha1 = "a630cfb0c2018a616d52f30e34cffa071ce93d42" uuid = "3c9796d7-64a0-5134-86ad-79f8eb684845" -version = "0.4.5+4" +version = "0.4.6+0" [[deps.Xorg_libXcursor_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"] -git-tree-sha1 = "807c226eaf3651e7b2c468f687ac788291f9a89b" +git-tree-sha1 = "6c74ca84bbabc18c4547014765d194ff0b4dc9da" uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724" -version = "1.2.3+0" +version = "1.2.4+0" [[deps.Xorg_libXdamage_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll"] -git-tree-sha1 = "fe4ffb2024ba3eddc862c6e1d70e2b070cd1c2bf" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXfixes_jll"] +git-tree-sha1 = "534ed7d299469f3438b2c136d7beb0b50da88ce0" uuid = "0aeada51-83db-5f97-b67e-184615cfc6f6" -version = "1.1.5+4" +version = "1.1.6+0" [[deps.Xorg_libXdmcp_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "89799ae67c17caa5b3b5a19b8469eeee474377db" +git-tree-sha1 = "52858d64353db33a56e13c341d7bf44cd0d7b309" uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" -version = "1.1.5+0" +version = "1.1.6+0" [[deps.Xorg_libXext_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] -git-tree-sha1 = "d7155fea91a4123ef59f42c4afb5ab3b4ca95058" +git-tree-sha1 = "a4c0ee07ad36bf8bbce1c3bb52d21fb1e0b987fb" uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" -version = "1.3.6+3" +version = "1.3.7+0" [[deps.Xorg_libXfixes_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] -git-tree-sha1 = "6fcc21d5aea1a0b7cce6cab3e62246abd1949b86" +git-tree-sha1 = "9caba99d38404b285db8801d5c45ef4f4f425a6d" uuid = "d091e8ba-531a-589c-9de9-94069b037ed8" -version = "6.0.0+0" +version = "6.0.1+0" [[deps.Xorg_libXi_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXext_jll", "Xorg_libXfixes_jll"] -git-tree-sha1 = "984b313b049c89739075b8e2a94407076de17449" +git-tree-sha1 = "a376af5c7ae60d29825164db40787f15c80c7c54" uuid = "a51aa0fd-4e3c-5386-b890-e753decda492" -version = "1.8.2+0" +version = "1.8.3+0" [[deps.Xorg_libXinerama_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXext_jll"] -git-tree-sha1 = "a1a7eaf6c3b5b05cb903e35e8372049b107ac729" +git-tree-sha1 = "a5bc75478d323358a90dc36766f3c99ba7feb024" uuid = "d1454406-59df-5ea1-beac-c340f2130bc3" -version = "1.1.5+0" +version = "1.1.6+0" [[deps.Xorg_libXrandr_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXext_jll", "Xorg_libXrender_jll"] -git-tree-sha1 = "b6f664b7b2f6a39689d822a6300b14df4668f0f4" +git-tree-sha1 = "aff463c82a773cb86061bce8d53a0d976854923e" uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484" -version = "1.5.4+0" +version = "1.5.5+0" [[deps.Xorg_libXrender_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] -git-tree-sha1 = "a490c6212a0e90d2d55111ac956f7c4fa9c277a6" +git-tree-sha1 = "7ed9347888fac59a618302ee38216dd0379c480d" uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" -version = "0.9.11+1" +version = "0.9.12+0" [[deps.Xorg_libXtst_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXfixes_jll", "Xorg_libXi_jll"] -git-tree-sha1 = "0c0a60851f44add2a64069ddf213e941c30ed93c" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXext_jll", "Xorg_libXfixes_jll", "Xorg_libXi_jll"] +git-tree-sha1 = "58c2e8f49733034d33b461d45a7e874b4135b644" uuid = "b6f176f1-7aea-5357-ad67-1d3e565ea1c6" -version = "1.2.3+4" - -[[deps.Xorg_libpthread_stubs_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "c57201109a9e4c0585b208bb408bc41d205ac4e9" -uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" -version = "0.1.2+0" +version = "1.2.5+0" [[deps.Xorg_libxcb_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] -git-tree-sha1 = "1a74296303b6524a0472a8cb12d3d87a78eb3612" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libXau_jll", "Xorg_libXdmcp_jll"] +git-tree-sha1 = "bfcaf7ec088eaba362093393fe11aa141fa15422" uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" -version = "1.17.0+3" +version = "1.17.1+0" [[deps.Xorg_libxkbfile_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] -git-tree-sha1 = "dbc53e4cf7701c6c7047c51e17d6e64df55dca94" +git-tree-sha1 = "e3150c7400c41e207012b41659591f083f3ef795" uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a" -version = "1.1.2+1" +version = "1.1.3+0" + +[[deps.Xorg_xcb_util_cursor_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_jll", "Xorg_xcb_util_renderutil_jll"] +git-tree-sha1 = "c5bf2dad6a03dfef57ea0a170a1fe493601603f2" +uuid = "e920d4aa-a673-5f3a-b3d7-f755a4d47c43" +version = "0.1.5+0" [[deps.Xorg_xcb_util_image_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] -git-tree-sha1 = "0fab0a40349ba1cba2c1da699243396ff8e94b97" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_jll"] +git-tree-sha1 = "f4fc02e384b74418679983a97385644b67e1263b" uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b" -version = "0.4.0+1" +version = "0.4.1+0" [[deps.Xorg_xcb_util_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll"] -git-tree-sha1 = "e7fd7b2881fa2eaa72717420894d3938177862d1" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll"] +git-tree-sha1 = "68da27247e7d8d8dafd1fcf0c3654ad6506f5f97" uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5" -version = "0.4.0+1" +version = "0.4.1+0" [[deps.Xorg_xcb_util_keysyms_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] -git-tree-sha1 = "d1151e2c45a544f32441a567d1690e701ec89b00" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_jll"] +git-tree-sha1 = "44ec54b0e2acd408b0fb361e1e9244c60c9c3dd4" uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7" -version = "0.4.0+1" +version = "0.4.1+0" [[deps.Xorg_xcb_util_renderutil_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] -git-tree-sha1 = "dfd7a8f38d4613b6a575253b3174dd991ca6183e" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_jll"] +git-tree-sha1 = "5b0263b6d080716a02544c55fdff2c8d7f9a16a0" uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e" -version = "0.3.9+1" +version = "0.3.10+0" [[deps.Xorg_xcb_util_wm_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] -git-tree-sha1 = "e78d10aab01a4a154142c5006ed44fd9e8e31b67" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xcb_util_jll"] +git-tree-sha1 = "f233c83cad1fa0e70b7771e0e21b061a116f2763" uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361" -version = "0.4.1+1" +version = "0.4.2+0" [[deps.Xorg_xkbcomp_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxkbfile_jll"] -git-tree-sha1 = "ab2221d309eda71020cdda67a973aa582aa85d69" +git-tree-sha1 = "801a858fc9fb90c11ffddee1801bb06a738bda9b" uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4" -version = "1.4.6+1" +version = "1.4.7+0" [[deps.Xorg_xkeyboard_config_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_xkbcomp_jll"] -git-tree-sha1 = "691634e5453ad362044e2ad653e79f3ee3bb98c3" +git-tree-sha1 = "00af7ebdc563c9217ecc67776d1bbf037dbcebf4" uuid = "33bec58e-1273-512f-9401-5d533626f822" -version = "2.39.0+0" +version = "2.44.0+0" [[deps.Xorg_xtrans_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6dba04dbfb72ae3ebe5418ba33d087ba8aa8cb00" +git-tree-sha1 = "a63799ff68005991f9d9491b6e95bd3478d783cb" uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" -version = "1.5.1+0" +version = "1.6.0+0" [[deps.YAML]] deps = ["Base64", "Dates", "Printf", "StringEncodings"] -git-tree-sha1 = "dea63ff72079443240fbd013ba006bcbc8a9ac00" +git-tree-sha1 = "2f58ac39f64b41fb812340347525be3b590cce3b" uuid = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" -version = "0.4.12" +version = "0.4.14" [[deps.Zlib_jll]] deps = ["Libdl"] @@ -2583,39 +2690,45 @@ version = "1.2.13+1" [[deps.Zstd_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "622cf78670d067c738667aaa96c553430b65e269" +git-tree-sha1 = "446b23e73536f84e8037f5dce465e92275f6a308" uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" -version = "1.5.7+0" +version = "1.5.7+1" [[deps.at_spi2_atk_jll]] -deps = ["ATK_jll", "Artifacts", "JLLWrappers", "Libdl", "Pkg", "XML2_jll", "Xorg_libX11_jll", "at_spi2_core_jll"] -git-tree-sha1 = "f16ae690aca4761f33d2cb338ee9899e541f5eae" +deps = ["ATK_jll", "Artifacts", "JLLWrappers", "Libdl", "XML2_jll", "Xorg_libX11_jll", "at_spi2_core_jll"] +git-tree-sha1 = "249ab7c4da34d86d3d66b524705b1c07fe96b815" uuid = "de012916-1e3f-58c2-8f29-df3ef51d412d" -version = "2.34.1+4" +version = "2.38.0+0" [[deps.at_spi2_core_jll]] -deps = ["Artifacts", "Dbus_jll", "Glib_jll", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXtst_jll"] -git-tree-sha1 = "d2d540cd145f2b2933614649c029d222fe125188" +deps = ["Artifacts", "Dbus_jll", "Glib_jll", "JLLWrappers", "Libdl", "Xorg_libXtst_jll"] +git-tree-sha1 = "120f41bc9540b8f137e5e5dea65845ee4f089f9e" uuid = "0fc3237b-ac94-5853-b45c-d43d59a06200" -version = "2.34.0+4" +version = "2.57.1+0" + +[[deps.eudev_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "c3b0e6196d50eab0c5ed34021aaa0bb463489510" +uuid = "35ca27e7-8b34-5b7f-bca9-bdc33f59eb06" +version = "3.2.14+0" [[deps.fzf_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6e50f145003024df4f5cb96c7fce79466741d601" +git-tree-sha1 = "b6a34e0e0960190ac2a4363a1bd003504772d631" uuid = "214eeab7-80f7-51ab-84ad-2988db7cef09" -version = "0.56.3+0" +version = "0.61.1+0" [[deps.gdk_pixbuf_jll]] -deps = ["Artifacts", "Glib_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pkg", "Xorg_libX11_jll", "libpng_jll"] -git-tree-sha1 = "e9190f9fb03f9c3b15b9fb0c380b0d57a3c8ea39" +deps = ["Artifacts", "Glib_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Xorg_libX11_jll", "libpng_jll"] +git-tree-sha1 = "895f21b699121d1a57ecac57e65a852caf569254" uuid = "da03df04-f53b-5353-a52f-6a8b0620ced0" -version = "2.42.8+0" +version = "2.42.13+0" [[deps.iso_codes_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d837a5d2a19d54243dafb6cc98d0f590a603dfa1" +git-tree-sha1 = "4d295b7797afbe24ad5d0452f673b901a81c43c3" uuid = "bf975903-5238-5d20-8243-bc370bc1e7e5" -version = "4.15.1+0" +version = "4.17.0+0" [[deps.isoband_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -2633,15 +2746,15 @@ version = "0.0.1-DEV" [[deps.libaom_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "522c1df09d05a71785765d19c9524661234738e9" +git-tree-sha1 = "4bba74fa59ab0755167ad24f98800fe5d727175b" uuid = "a4ae2306-e953-59d6-aa16-d00cac43593b" -version = "3.11.0+0" +version = "3.12.1+0" [[deps.libass_jll]] deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "e17c115d55c5fbb7e52ebedb427a0dca79d4484e" +git-tree-sha1 = "125eedcb0a4a0bba65b657251ce1d27c8714e9d6" uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" -version = "0.15.2+0" +version = "0.17.4+0" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] @@ -2654,17 +2767,29 @@ git-tree-sha1 = "9bf7903af251d2050b467f76bdbe57ce541f7f4f" uuid = "1183f4f0-6f2a-5f1a-908b-139f9cdfea6f" version = "0.2.2+0" +[[deps.libevdev_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "56d643b57b188d30cccc25e331d416d3d358e557" +uuid = "2db6ffa8-e38f-5e21-84af-90c45d0032cc" +version = "1.13.4+0" + [[deps.libfdk_aac_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "8a22cf860a7d27e4f3498a0fe0811a7957badb38" +git-tree-sha1 = "646634dd19587a56ee2f1199563ec056c5f228df" uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" -version = "2.0.3+0" +version = "2.0.4+0" + +[[deps.libinput_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "eudev_jll", "libevdev_jll", "mtdev_jll"] +git-tree-sha1 = "91d05d7f4a9f67205bd6cf395e488009fe85b499" +uuid = "36db933b-70db-51c0-b978-0f229ee0e533" +version = "1.28.1+0" [[deps.libpng_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "d7b5bbf1efbafb5eca466700949625e07533aff2" +git-tree-sha1 = "07b6a107d926093898e82b3b1db657ebe33134ec" uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" -version = "1.6.45+1" +version = "1.6.50+0" [[deps.libsixel_jll]] deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "libpng_jll"] @@ -2673,16 +2798,28 @@ uuid = "075b6546-f08a-558a-be8f-8157d0f608a5" version = "1.10.5+0" [[deps.libvorbis_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] -git-tree-sha1 = "490376214c4721cdaca654041f635213c6165cb3" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll"] +git-tree-sha1 = "11e1772e7f3cc987e9d3de991dd4f6b2602663a5" uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" -version = "1.3.7+2" +version = "1.3.8+0" [[deps.libwebp_jll]] deps = ["Artifacts", "Giflib_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libglvnd_jll", "Libtiff_jll", "libpng_jll"] -git-tree-sha1 = "ccbb625a89ec6195856a50aa2b668a5c08712c94" +git-tree-sha1 = "4e4282c4d846e11dce56d74fa8040130b7a95cb3" uuid = "c5f90fcd-3b7e-5836-afba-fc50a0988cb2" -version = "1.4.0+0" +version = "1.6.0+0" + +[[deps.libzip_jll]] +deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "OpenSSL_jll", "XZ_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "86addc139bca85fdf9e7741e10977c45785727b7" +uuid = "337d8026-41b4-5cde-a456-74a10e5b31d1" +version = "1.11.3+0" + +[[deps.mtdev_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b4d631fd51f2e9cdd93724ae25b2efc198b059b1" +uuid = "009596ad-96f7-51b1-9f1b-5ce2d5e8a71e" +version = "1.1.7+0" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] @@ -2691,9 +2828,9 @@ version = "1.59.0+0" [[deps.oneTBB_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "7d0ea0f4895ef2f5cb83645fa689e52cb55cf493" +git-tree-sha1 = "d5a767a3bb77135a99e433afe0eb14cd7f6914c3" uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e" -version = "2021.12.0+0" +version = "2022.0.0+0" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] @@ -2701,19 +2838,19 @@ uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" version = "17.4.0+2" [[deps.x264_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "14cc7083fc6dff3cc44f2bc435ee96d06ed79aa7" uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a" -version = "2021.5.5+0" +version = "10164.0.1+0" [[deps.x265_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e7b67590c14d487e734dcb925924c5dc43ec85f3" uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76" -version = "3.5.0+0" +version = "4.1.0+0" [[deps.xkbcommon_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"] -git-tree-sha1 = "63406453ed9b33a0df95d570816d5366c92b7809" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"] +git-tree-sha1 = "fbf139bce07a534df0e699dbb5f5cc9346f95cc1" uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd" -version = "1.4.1+2" +version = "1.9.2+0" diff --git a/Project.toml b/Project.toml index 2909ef6..653695f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,15 @@ +name = "MSI_src" +authors = ["JJSA"] +version = "0.1.0" + [deps] +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +ColorSchemes = "35d6a980-a343-548e-a6ea-1d62b119f2f4" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Genie = "c43c736e-a2d1-11e8-161f-af95117fbd1e" GenieFramework = "a59fdf5c-6bf0-4f5d-949c-a137c9e2f353" Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" @@ -9,6 +18,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4" NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85" PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StipplePlotly = "ec984513-233d-481d-95b0-a3b58b97af2b" julia_mzML_imzML = "38eb50d3-2fb6-4afa-992a-964ed8562ed9" diff --git a/app.jl b/app.jl index 4bf8124..b0fee09 100644 --- a/app.jl +++ b/app.jl @@ -6,7 +6,7 @@ using Libz using PlotlyBase using CairoMakie using Colors -# using julia_mzML_imzML +using julia_mzML_imzML using Statistics using NaturalSort using Images @@ -15,7 +15,7 @@ using NativeFileDialog # Opens the file explorer depending on the OS using StipplePlotly using Base.Filesystem: mv # To rename files in the system include("./julia_imzML_visual.jl") -include("./src/imzML.jl") + @genietools # == Reactive code == @@ -363,7 +363,7 @@ include("./src/imzML.jl") if isfile(full_route) && Nmass > 0 && Tol > 0 && Tol <=1 && colorLevel > 1 && colorLevel < 257 msg="File exists, Nmass=$(Nmass) Tol=$(Tol). Loading file will begin, please be patient." try - spectra=load_imzml(full_route) + spectra=LoadImzml(full_route) msg="File loaded. Creating spectra with the specific mass and tolerance, please be patient." slice=GetMzSliceJl(spectra,Nmass,Tol) fig=CairoMakie.Figure(size=(150, 250)) # Container diff --git a/src/Bitmap.jl b/src/Bitmap.jl deleted file mode 100644 index a7edbf1..0000000 --- a/src/Bitmap.jl +++ /dev/null @@ -1,399 +0,0 @@ -# ******************************************************************** -# SaveBitmap -# ******************************************************************** -""" - SaveBitmap( name, pixMap, colorTable ) - -Save a discretized imzML slice as a bitmap file - -# Arguments -* ` name`: Full path name of target bitmap -* ` pixMap`: [UInt8] Bidimensional matrix with image info -* `colorTable`: [UInt32] Vector with RGB colors for each gray level - -# Examples -```julia -# Saves a bitmap with black and white alternate image squares -img = zeros(UInt8, 32, 32) -img[17:32,1:16] .= 1 -img[1:16,17:32] .= 1 -SaveBitmap( "test.bmp", img, UInt32[0, 0x00FFFFFF] ) -``` - -The following image will be created on your hard disk - -![](./Test.bmp) -""" -function SaveBitmap( name, - pixMap::Array{UInt8,2}, - colorTable::Array{UInt32,1} ) - - # Get image dimensions - dim = size( pixMap ) - if length( dim ) != 2 - return 0 - end - - # Compute row padding - padding = ( 4 - dim[1] & 0x3 ) & 0x3 - - # Compute file dimensions. Header = 14 + 40 + ( 256 * 4 ) = 1078 - offset = 1078 - imgBytes = dim[2] * ( dim[1] + padding ) - - # Create file - stream = open( name, "w" ) - - # Save file header - write( stream, UInt16( 0x4D42 ) ) - write( stream, UInt32[ offset + imgBytes, 0 , offset ] ) - - # Save info header - write( stream, UInt32[ 40, dim[1], dim[2], 0x80001, 0 ] ) - write( stream, UInt32[ imgBytes, 0, 0, 256, 0 ] ) - - # Save color table - write( stream, colorTable ) - if length( colorTable ) < 256 - fixTable = zeros( UInt32, 256 - length( colorTable ) ) - write( stream, fixTable ) - end - - # Save image pixels - if padding == 0 - for i = 1:dim[2] - write( stream, pixMap[:,i] ) - end - else - zeroPad = zeros( UInt8, padding ) - for i in 1:dim[2] - write( stream, pixMap[:,i] ) - write( stream, zeroPad ) - end - end - - # Close file - close( stream ) - -end - - -# ******************************************************************** -# FindMass -# massVector: mz Vector sorted in ascending order -# mass: target mz value -# tolerance: bi-axial tolerance to find mass -# ******************************************************************** -function FindMass( massVector, mass, tolerance ) - - index = Int( 0 ) - lower = Int( 1 ) - higher = Int( length( massVector ) ) - - while lower <= higher - - # Compute mid element - index = ( lower + higher ) ÷ 2 - - # Go to lower portion ? - if massVector[ index ] > ( mass + tolerance ) - higher = index - 1 - continue - end - - # Go to Higher portion? - if massVector[ index ] < ( mass - tolerance ) - lower = index + 1 - continue - end - - # mass was found - return index - end - - # mass was not found - return 0 -end - - -# ******************************************************************** -# GetSlice -# imzML: imzML data -# mass: mz target value -# tolerance: bi-axial tolerance to find mass -# ******************************************************************** -""" - GetSlice( imzML, mass, tolerance ) - -Extract a mz-image from an imzML image data array loaded with - `LoadImzml` function. The resulting image array preserves the same - data type of the y-axis vectors in the file. - -# Arguments -* ` imzML`: Image array loaded with `LoadImzml` function -* ` mass`: mz value for image extraction -* `tolerance`: Maximum lateral tolerance for a valid mz search - -# Examples -```julia -# Load DESI MSI Carcinoma 885.55 mz slice -spectra = LoadImzml( "80TopL, 50TopR, 70BottomL, 60BottomR-centroid.imzML" ) -slice = GetSlice( spectra, 885.55, 0.005 ) -``` -""" -function GetSlice( imzML, mass, tolerance ) - - # Alloc space for slice - width = maximum( imzML[1,:] ) - height = maximum( imzML[2,:] ) - image = zeros( Float64, width, height ) - - for i in 1:size( imzML )[2] - index = FindMass( imzML[3,i], mass, tolerance ) - if index != 0 - image[ imzML[1,i], imzML[2,i] ] = imzML[4,i][index] - end - end - - return image - -end - - -# ******************************************************************** -# IntQuant, Discretize image amplitude in 0:255 range -# slice: Image matriz in measured arbitrary units -# ******************************************************************** -""" - IntQuant( slice ) - -Zero Memory intensity quantizer. Discretize the image?s continuos - intensity range in discrete bins from 0 to 255 gray levels. - Returns a UInt8 bidimensional matrix - -# Arguments -* `slice`: slice returned by the `GetSlice` function - -# Examples -```julia -# Save DESI MSI Carcinoma 885.55 mz slice as a bitmap file -spectra = LoadImzml( "80TopL, 50TopR, 70BottomL, 60BottomR-centroid.imzML" ) -slice = GetSlice( spectra, 885.55, 0.005 -SaveBitmap( "Slice.bmp", IntQuant( slice ), ViridisPalette ) -``` - -The following image will be created on your hard disk - -![](./Slice.bmp) -""" -function IntQuant( slice ) - - # Compute scale factor for amplitude discretization - lower = minimum( slice ) - scale = 255 / maximum( slice ) - dim = size( slice ) - image = zeros( UInt8, dim[1], dim[2] ) - - for i in 1:length( slice ) - image[i] = convert( UInt8, floor( slice[i] * scale + 0.5 ) ) - end - - return image - -end - - - -# ******************************************************************** -# TrIQ, Discretize image amplitude in 0:255 range, grouping outliers -# in the highest bin -# slice: Image matrix in measured arbitrary units -# ******************************************************************** -function GetOutlierThres( slice, prob ) - - # Get bin width - low = minimum( slice ) - upp = maximum( slice ) - - # Compute histogram's bin count & separation - if ( upp - low + 1 ) >= 100 - bins = convert( Int32, 100 ) - step = ( upp + (upp-low)/(bins-1) - low ) / bins - else - bins = convert( Int32, ceil( upp ) - floor( low ) + 1 ) - step = 1 - end - - # Compute histogram - histCount = zeros( bins, 1 ) - for k in slice - index = convert( Int64, floor( ( k - low ) / step ) + 1 ) - histCount[ index ] += 1 - end - - # Get bin index, that accounts for desired probability - prob = 0.95 - delta = cumsum( histCount, dims=1 ) / sum( histCount ) .- prob - index = findmin( broadcast( abs, delta ) )[2][1] - - # Find max intensity image value - key = low + step * index - upp = -1 - - for k in slice[:] - if key > k && k > upp - upp = k - end - end - - return [ low, upp ] - -end - - -# ******************************************************************** -# Discretize image's intensity using a given intensity range -# slice: Image matrix in measured arbitrary units -# bounds: max and min intensity discretizing range -# depth: number of intensity discrete steps in final image -# ******************************************************************** -function SetPixelDepth( slice, bounds, depth ) - - # Compute intensity bins - bins = depth - 1 - limits = collect( - range( bounds[1], stop = bounds[2], length = depth ) ) - - # Reserve memory for output image - imgBytes = zeros( UInt8, size( slice ) ) - nPixels = length( slice ) - step = limits[2] - limits[1] - - # Set intensity depth - for k in 1:nPixels - - # Find bin - i::Int64 = floor( ( slice[k] - limits[1] ) / step ) + 1 - if i < bins - imgBytes[k] = i + ( slice[k] > limits[i] ) - 1 - else - imgBytes[k] = bins - end - - end - - return imgBytes -end - - -# ******************************************************************** -# Discretize image's intensity removing intensity outliers -# slice: Image matrix in measured arbitrary units -# depth: Number of intensity discrete steps in final image -# prob: Proportion of pixesl to take into account in discretization -# ******************************************************************** -""" - TrIQ( slice, depth, prob = 0.98 ) - -TrIQ intensity quantizer as described in _DOI 10.7717/peerj-cs.585_. - Discretize the image?s continuos intensity range in discrete bins - from 0 to 255 gray levels, groupping intensity outiers in the - highehst bin. Returns a UInt8 bidimensional matrix - -# Arguments -* `slice`: slice returned by the `GetSlice` function -* `depth`: Number ob discrete bins for intensity quantization -* ` prob`: Cummulative distribute function cutting value - -# Examples -```julia -# Save DESI MSI Carcinoma 885.55 mz slice as a bitmap file -spectra = LoadImzml( "80TopL, 50TopR, 70BottomL, 60BottomR-centroid.imzML" ) -slice = GetSlice( spectra, 885.55, 0.005 ) -SaveBitmap( "TrIQ.bmp", TrIQ( slice, 256, 0.95 ), ViridisPalette ) -``` - -The following image will be created on your hard disk - -![](./TrIQ.bmp) -""" -function TrIQ( slice, depth, prob ) - - return SetPixelDepth( - slice, - GetOutlierThres( slice, prob ), - depth - ) - -end - - -# ******************************************************************** -# Viridis color palette 256 color levels -# ******************************************************************** -ViridisPalette = UInt32[ - 0x440154, 0x440256, 0x450457, 0x450559, - 0x46075A, 0x46085C, 0x460A5D, 0x460B5E, - 0x470D60, 0x470E61, 0x471063, 0x471164, - 0x471365, 0x481467, 0x481668, 0x481769, - 0x48186A, 0x481A6C, 0x481B6D, 0x481C6E, - 0x481D6F, 0x481F70, 0x482071, 0x482173, - 0x482374, 0x482475, 0x482576, 0x482677, - 0x482878, 0x482979, 0x472A7A, 0x472C7A, - 0x472D7B, 0x472E7C, 0x472F7D, 0x46307E, - 0x46327E, 0x46337F, 0x463480, 0x453581, - 0x453781, 0x453882, 0x443983, 0x443A83, - 0x443B84, 0x433D84, 0x433E85, 0x423F85, - 0x424086, 0x424186, 0x414287, 0x414487, - 0x404588, 0x404688, 0x3F4788, 0x3F4889, - 0x3E4989, 0x3E4A89, 0x3E4C8A, 0x3D4D8A, - 0x3D4E8A, 0x3C4F8A, 0x3C508B, 0x3B518B, - 0x3B528B, 0x3A538B, 0x3A548C, 0x39558C, - 0x39568C, 0x38588C, 0x38598C, 0x375A8C, - 0x375B8D, 0x365C8D, 0x365D8D, 0x355E8D, - 0x355F8D, 0x34608D, 0x34618D, 0x33628D, - 0x33638D, 0x32648E, 0x32658E, 0x31668E, - 0x31678E, 0x31688E, 0x30698E, 0x306A8E, - 0x2F6B8E, 0x2F6C8E, 0x2E6D8E, 0x2E6E8E, - 0x2E6F8E, 0x2D708E, 0x2D718E, 0x2C718E, - 0x2C728E, 0x2C738E, 0x2B748E, 0x2B758E, - 0x2A768E, 0x2A778E, 0x2A788E, 0x29798E, - 0x297A8E, 0x297B8E, 0x287C8E, 0x287D8E, - 0x277E8E, 0x277F8E, 0x27808E, 0x26818E, - 0x26828E, 0x26828E, 0x25838E, 0x25848E, - 0x25858E, 0x24868E, 0x24878E, 0x23888E, - 0x23898E, 0x238A8D, 0x228B8D, 0x228C8D, - 0x228D8D, 0x218E8D, 0x218F8D, 0x21908D, - 0x21918C, 0x20928C, 0x20928C, 0x20938C, - 0x1F948C, 0x1F958B, 0x1F968B, 0x1F978B, - 0x1F988B, 0x1F998A, 0x1F9A8A, 0x1E9B8A, - 0x1E9C89, 0x1E9D89, 0x1F9E89, 0x1F9F88, - 0x1FA088, 0x1FA188, 0x1FA187, 0x1FA287, - 0x20A386, 0x20A486, 0x21A585, 0x21A685, - 0x22A785, 0x22A884, 0x23A983, 0x24AA83, - 0x25AB82, 0x25AC82, 0x26AD81, 0x27AD81, - 0x28AE80, 0x29AF7F, 0x2AB07F, 0x2CB17E, - 0x2DB27D, 0x2EB37C, 0x2FB47C, 0x31B57B, - 0x32B67A, 0x34B679, 0x35B779, 0x37B878, - 0x38B977, 0x3ABA76, 0x3BBB75, 0x3DBC74, - 0x3FBC73, 0x40BD72, 0x42BE71, 0x44BF70, - 0x46C06F, 0x48C16E, 0x4AC16D, 0x4CC26C, - 0x4EC36B, 0x50C46A, 0x52C569, 0x54C568, - 0x56C667, 0x58C765, 0x5AC864, 0x5CC863, - 0x5EC962, 0x60CA60, 0x63CB5F, 0x65CB5E, - 0x67CC5C, 0x69CD5B, 0x6CCD5A, 0x6ECE58, - 0x70CF57, 0x73D056, 0x75D054, 0x77D153, - 0x7AD151, 0x7CD250, 0x7FD34E, 0x81D34D, - 0x84D44B, 0x86D549, 0x89D548, 0x8BD646, - 0x8ED645, 0x90D743, 0x93D741, 0x95D840, - 0x98D83E, 0x9BD93C, 0x9DD93B, 0xA0DA39, - 0xA2DA37, 0xA5DB36, 0xA8DB34, 0xAADC32, - 0xADDC30, 0xB0DD2F, 0xB2DD2D, 0xB5DE2B, - 0xB8DE29, 0xBADE28, 0xBDDF26, 0xC0DF25, - 0xC2DF23, 0xC5E021, 0xC8E020, 0xCAE11F, - 0xCDE11D, 0xD0E11C, 0xD2E21B, 0xD5E21A, - 0xD8E219, 0xDAE319, 0xDDE318, 0xDFE318, - 0xE2E418, 0xE5E419, 0xE7E419, 0xEAE51A, - 0xECE51B, 0xEFE51C, 0xF1E51D, 0xF4E61E, - 0xF6E620, 0xF8E621, 0xFBE723, 0xFDE725 ] - diff --git a/src/Common.jl b/src/Common.jl deleted file mode 100644 index aa26b94..0000000 --- a/src/Common.jl +++ /dev/null @@ -1,94 +0,0 @@ -# ******************************************************************* -# VectorConfig, structure with axis decoding instructions -# ******************************************************************* -mutable struct SpecDim - Format::DataType # Float32, Float64 - Packed::Bool # 0: none 1: zlib compression - Axis::Int32 # 1: m/z 2: Amplitude - Skip::Int64 # Bytes to skip -end - -# ******************************************************************* -# Fill VectorConfig fields from "cvParam" tags -# ******************************************************************* -function ConfigureSpecDim( stream ) - - # Vector configuration - axis = SpecDim( Float64, false, 1, 0 ) - - # Initial values - offset = position( stream ) - currLine = "" - matchInfo = RegexMatch - - while true - - # Next field - currLine = readline( stream ) - matchInfo = match( r"^\s*<(cvParam)", currLine ) - - if matchInfo === nothing - matchInfo = match( r"^\s*", currLine ) - axis.Skip = position( stream ) - offset - - length( currLine ) + length( matchInfo.match ) - return axis - end - - index = length( matchInfo.captures[1] ) - matchInfo = GetAttribute( currLine[index:end], "accession" ) - - if matchInfo.captures[1] == "MS:1000515" # intensity array - axis.Axis = 2 - continue - - elseif matchInfo.captures[1] == "MS:1000519" # 32-bit integer - axis.Format = Int32 - continue - - elseif matchInfo.captures[1] == "MS:1000521" # 32-bit float - axis.Format = Float32 - continue - - elseif matchInfo.captures[1] == "MS:1000522" # 64-bit integer - axis.Format = Int64 - continue - - elseif matchInfo.captures[1] == "MS:1000574" # zlib compresion - axis.Packed = true - continue - - end - end -end - - -# ******************************************************************* -# Read lines in file until regex match -# stream: Source file stream -# regex: target tag e.g. "binaryDataArray" -# ******************************************************************* -function FindTag( stream, regex ) - - while true - isTag = match( regex, readline( stream ) ) - if isTag !== nothing - return isTag - end - end -end - - -# ******************************************************************* -# Retrieve attribute/value pair -# source: String with atr="value" pairs -# tag: Optional attribute, e.g. "\\spectrum" -# ******************************************************************* -function GetAttribute( source, tag = "([^=]+)" ) - - # Build regex and retrieve attribute/value pair - regStr = Regex( "\\s" * tag * "=\"([^\"]*)\"" ) - return( match( regStr, source ) ) - -end - - diff --git a/src/DataAccess.jl b/src/DataAccess.jl deleted file mode 100644 index 5977248..0000000 --- a/src/DataAccess.jl +++ /dev/null @@ -1,82 +0,0 @@ -# src/DataAccess.jl - -""" - load_spectra(fileName::String) - -Eagerly loads all spectra from a .mzML or .imzML file into a 2xN matrix. -This function follows the eager-loading pattern of the original `LoadMzml` -for backward compatibility and for analyses that require all data in memory. - -Each column in the returned matrix represents a single spectrum: -- Row 1: m/z array -- Row 2: Intensity array - -For .imzML files, the (x,y) spatial coordinates are discarded. - -# Arguments -* `fileName`: Full path to the .mzML or .imzML file. - -# Returns -- An `Array{Any, 2}` containing all spectra. -""" -function load_spectra(fileName::String) - if endswith(lowercase(fileName), ".mzml") - # Detected .mzML file. Using the existing eager loader. - return LoadMzml(fileName) - - elseif endswith(lowercase(fileName), ".imzml") - # Detected .imzML file. Eagerly loading all spectra from .ibd. - - # Lazily load metadata to get offsets and counts - imzml_data = load_imzml(fileName) - - try - num_spectra = length(imzml_data.spectra_metadata) - if num_spectra == 0 - return Array{Any}(undef, (2, 0)) - end - - spectra_matrix = Array{Any}(undef, (2, num_spectra)) - - hIbd = imzml_data.ibd_handle - mz_format = imzml_data.mz_format - int_format = imzml_data.intensity_format - - # For performance, pre-allocate one large buffer for reading. - # This avoids re-allocating memory for each spectrum inside the loop. - max_points = maximum(meta.mz_count for meta in imzml_data.spectra_metadata) - mz_array_buffer = Array{mz_format}(undef, max_points) - intensity_array_buffer = Array{int_format}(undef, max_points) - - for i in 1:num_spectra - meta = imzml_data.spectra_metadata[i] - - # Create a view into the buffer with the correct size for this spectrum - current_mz_array = view(mz_array_buffer, 1:meta.mz_count) - current_int_array = view(intensity_array_buffer, 1:meta.intensity_count) - - # Read binary data directly into the sized views - seek(hIbd, meta.mz_offset) - read!(hIbd, current_mz_array) - - seek(hIbd, meta.intensity_offset) - read!(hIbd, current_int_array) - - # Store a copy in the final matrix. A copy is necessary because - # the buffer is overwritten in the next iteration. - spectra_matrix[1, i] = copy(current_mz_array) - spectra_matrix[2, i] = copy(current_int_array) - end - - return spectra_matrix - finally - # Ensure the .ibd file handle is closed, as this is an eager load. - if isopen(imzml_data.ibd_handle) - close(imzml_data.ibd_handle) - end - end - - else - error("Unsupported file type. Please provide a .mzML or .imzML file.") - end -end diff --git a/src/MSIData.jl b/src/MSIData.jl new file mode 100644 index 0000000..4ac7107 --- /dev/null +++ b/src/MSIData.jl @@ -0,0 +1,369 @@ +# src/MSIData.jl + +""" +This file defines the unified MSIData object and the associated data access layer, +including caching and iteration logic, for handling large mzML and imzML datasets +efficiently. +""" + +using Base64, Libz # For reading binary data + +# Abstract type for different data sources (e.g., mzML, imzML) +# This allows dispatching to the correct binary reading logic. +abstract type MSDataSource end + +# Concrete type for imzML data sources +struct ImzMLSource <: MSDataSource + ibd_handle::IO + mz_format::Type + intensity_format::Type +end + +# Concrete type for mzML data sources +struct MzMLSource <: MSDataSource + file_handle::IO + mz_format::Type + intensity_format::Type +end + +# Struct to hold info about a binary data array (mz or intensity) +struct SpectrumAsset + format::Type + is_compressed::Bool + offset::Int64 + encoded_length::Int32 + # For mzML, axis_type is needed to distinguish mz from intensity. + # For imzML, this can be ignored as the order is fixed. + axis_type::Symbol +end + +# Metadata for a single spectrum, common to both formats +struct SpectrumMetadata + # For imzML + x::Int32 + y::Int32 + + # For mzML + id::String + + # Common binary data info + mz_asset::SpectrumAsset + int_asset::SpectrumAsset +end + +# The main unified data object +mutable struct MSIData + source::MSDataSource + spectra_metadata::Vector{SpectrumMetadata} + image_dims::Tuple{Int, Int} # (width, height) for imaging data + coordinate_map::Union{Matrix{Int}, Nothing} # Maps (x,y) to linear index for imzML + + # LRU Cache implementation + cache::Dict{Int, Tuple{Vector, Vector}} + cache_order::Vector{Int} # Stores indices, with most recently used at the end + cache_size::Int # Max number of spectra in cache + + function MSIData(source, metadata, dims, coordinate_map, cache_size) + obj = new(source, metadata, dims, coordinate_map, Dict(), [], cache_size) + + # Ensure file handles are closed when the object is garbage collected + finalizer(obj) do o + if o.source isa ImzMLSource && isopen(o.source.ibd_handle) + close(o.source.ibd_handle) + elseif o.source isa MzMLSource && isopen(o.source.file_handle) + close(o.source.file_handle) + end + end + return obj + end +end + + +# --- Internal function for reading binary data --- # + +function read_binary_vector(io::IO, asset::SpectrumAsset) + seek(io, asset.offset) + raw_b64 = read(io, asset.encoded_length) + decoded_bytes = base64decode(strip(String(raw_b64))) + bytes_io = IOBuffer(asset.is_compressed ? Libz.inflate(decoded_bytes) : decoded_bytes) + + n_elements = Int(bytes_io.size / sizeof(asset.format)) + out_array = Array{asset.format}(undef, n_elements) + read!(bytes_io, out_array) + return out_array +end + +# Overload for different source types +function read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata) + # For imzML, the binary data is raw, not base64 encoded. + # The `encoded_length` field in this case holds the number of points. + mz = Array{source.mz_format}(undef, meta.mz_asset.encoded_length) + intensity = Array{source.intensity_format}(undef, meta.int_asset.encoded_length) + + seek(source.ibd_handle, meta.mz_asset.offset) + read!(source.ibd_handle, mz) + + seek(source.ibd_handle, meta.int_asset.offset) + read!(source.ibd_handle, intensity) + + return mz, intensity +end + +function read_spectrum_from_disk(source::MzMLSource, meta::SpectrumMetadata) + # For mzML, data is Base64 encoded within the XML + mz = read_binary_vector(source.file_handle, meta.mz_asset) + intensity = read_binary_vector(source.file_handle, meta.int_asset) + return mz, intensity +end + +# --- Public API --- # + +""" + GetSpectrum(data::MSIData, index::Int) + +Retrieves a single spectrum by its index, utilizing a cache for performance. + +This function is the core of the "Indexed" and "Cache" access patterns. +""" +function GetSpectrum(data::MSIData, index::Int) + if index < 1 || index > length(data.spectra_metadata) + error("Spectrum index $index out of bounds.") + end + + # Phase 1: Check the cache + if haskey(data.cache, index) + # Cache Hit: Move item to the end of the LRU list and return from cache + filter!(x -> x != index, data.cache_order) + push!(data.cache_order, index) + return data.cache[index] + end + + # Phase 2: Cache Miss - Read from disk + meta = data.spectra_metadata[index] + spectrum = read_spectrum_from_disk(data.source, meta) + + # Phase 3: Update cache + if data.cache_size > 0 + if length(data.cache) >= data.cache_size + # Evict the least recently used item (at the front of the list) + lru_index = popfirst!(data.cache_order) + delete!(data.cache, lru_index) + end + data.cache[index] = spectrum + push!(data.cache_order, index) + end + + return spectrum +end + +""" + GetSpectrum(data::MSIData, x::Int, y::Int) + +Retrieves a single spectrum by its (x, y) coordinates for imaging data. +Utilizes a coordinate map for efficient lookup and then the cache. +""" +function GetSpectrum(data::MSIData, x::Int, y::Int) + if data.coordinate_map === nothing + error("Coordinate map not available. This method is only for imaging data loaded from .imzML files.") + end + width, height = data.image_dims + if x < 1 || x > width || y < 1 || y > height + error("Coordinates ($x, $y) out of bounds for image dimensions ($width, $height).") + end + + index = data.coordinate_map[x, y] + if index == 0 + error("No spectrum found at coordinates ($x, $y).") + end + + return GetSpectrum(data, index) # Call the existing method +end + +""" + get_total_spectrum(msi_data::MSIData; num_bins::Int=20000) -> Tuple{Vector{Float64}, Vector{Float64}} + +Calculates the sum of all spectra in the dataset by binning. +The m/z range is determined dynamically by finding the global min/max m/z values +across the entire dataset. + +Returns a tuple containing two vectors: the binned m/z axis and the summed intensities. +""" +function get_total_spectrum(msi_data::MSIData; num_bins::Int=20000) + println("Calculating total spectrum...") + + # 1. First Pass: Find the global m/z range + println(" Pass 1: Finding global m/z range...") + global_min_mz = Inf + global_max_mz = -Inf + + _iterate_spectra_fast(msi_data) do idx, mz, _ + if !isempty(mz) + local_min, local_max = extrema(mz) + global_min_mz = min(global_min_mz, local_min) + global_max_mz = max(global_max_mz, local_max) + end + end + + if !isfinite(global_min_mz) + @warn "Could not determine a valid m/z range. All spectra might be empty." + return (Float64[], Float64[]) + end + println(" Global m/z range found: [$(global_min_mz), $(global_max_mz)]") + + # 2. Define Bins + mz_bins = range(global_min_mz, stop=global_max_mz, length=num_bins) + intensity_sum = zeros(Float64, num_bins) + bin_step = step(mz_bins) + + # 3. Second Pass: Sum intensities into bins + println(" Pass 2: Summing intensities into $num_bins bins...") + _iterate_spectra_fast(msi_data) do idx, mz, intensity + if isempty(mz) + return # continue equivalent + end + for i in eachindex(mz) + # Find the correct bin for the current m/z value + bin_index = clamp(round(Int, (mz[i] - global_min_mz) / bin_step) + 1, 1, num_bins) + intensity_sum[bin_index] += intensity[i] + end + end + + println("Total spectrum calculation complete.") + return (collect(mz_bins), intensity_sum) +end + + +""" + get_average_spectrum(msi_data::MSIData; num_bins::Int=20000) -> Tuple{Vector{Float64}, Vector{Float64}} + +Calculates the average of all spectra in the dataset by binning. +This is effectively the total ion chromatogram (TIC) divided by the number of spectra. + +Returns a tuple containing two vectors: the binned m/z axis and the averaged intensities. +""" +function get_average_spectrum(msi_data::MSIData; num_bins::Int=20000) + # This function uses the exact same logic as get_total_spectrum... + mz_bins, intensity_sum = get_total_spectrum(msi_data, num_bins=num_bins) + + if isempty(intensity_sum) + return (mz_bins, intensity_sum) + end + + # ...with one final step: dividing by the number of spectra to get the average. + println("Averaging spectrum...") + num_spectra = length(msi_data.spectra_metadata) + average_intensity = intensity_sum ./ num_spectra + + return (mz_bins, average_intensity) +end + +# --- Iterator Implementation --- # + +struct MSIDataIterator + data::MSIData +end + +""" + IterateSpectra(data::MSIData) + +Returns an iterator that yields each spectrum, processing the file sequentially +with minimal memory overhead. + +This function is the core of the "Event-driven" access pattern. +""" +function IterateSpectra(data::MSIData) + return MSIDataIterator(data) +end + +# Define the iteration interface for the custom iterator +Base.length(it::MSIDataIterator) = length(it.data.spectra_metadata) +Base.eltype(it::MSIDataIterator) = Tuple{Int, Tuple{Vector, Vector}} # Index, (mz, intensity) + +function Base.iterate(it::MSIDataIterator, state=1) + if state > length(it.data.spectra_metadata) + return nothing # End of iteration + end + + # GetSpectrum uses the cache, so iteration is also cached + spectrum = GetSpectrum(it.data, state) + + # Yield the index and the spectrum + return ((state, spectrum), state + 1) +end + + +# --- High-performance Internal Iterator --- # + +""" + _iterate_spectra_fast(f::Function, data::MSIData) + +An internal, high-performance iterator for bulk processing. +It avoids the overhead of `GetSpectrum` by reading directly from the file stream +and reusing pre-allocated buffers. + +- `f`: A function to call for each spectrum, with signature `f(index, mz, intensity)`. +- `data`: The `MSIData` object. +""" +function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::ImzMLSource) + # This implementation is for imzML and is optimized for performance by + # using pre-allocated buffers. + max_points = 0 + for meta in data.spectra_metadata + # For imzML, encoded_length is the number of points + max_points = max(max_points, meta.mz_asset.encoded_length) + end + + # If all spectra are empty, just call f with empty arrays + if max_points == 0 && !isempty(data.spectra_metadata) + for i in 1:length(data.spectra_metadata) + f(i, source.mz_format[], source.intensity_format[]) + end + return + end + + mz_buffer = Vector{source.mz_format}(undef, max_points) + int_buffer = Vector{source.intensity_format}(undef, max_points) + + for i in 1:length(data.spectra_metadata) + meta = data.spectra_metadata[i] + nPoints = meta.mz_asset.encoded_length + + if nPoints == 0 + f(i, source.mz_format[], source.intensity_format[]) + continue + end + + mz_view = view(mz_buffer, 1:nPoints) + int_view = view(int_buffer, 1:nPoints) + + seek(source.ibd_handle, meta.mz_asset.offset) + read!(source.ibd_handle, mz_view) + + seek(source.ibd_handle, meta.int_asset.offset) + read!(source.ibd_handle, int_view) + + f(i, mz_view, int_view) + end +end + +function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::MzMLSource) + # This implementation is for mzML. It iterates through each spectrum, + # decodes it, and then calls the function `f`. It's less performant than + # the imzML version because we cannot pre-allocate buffers of a known size, + # but it is correct and still faster than using GetSpectrum due to bypassing the cache. + for i in 1:length(data.spectra_metadata) + meta = data.spectra_metadata[i] + + mz = read_binary_vector(source.file_handle, meta.mz_asset) + intensity = read_binary_vector(source.file_handle, meta.int_asset) + + f(i, mz, intensity) + end +end + +function _iterate_spectra_fast(f::Function, data::MSIData) + # Dispatch to the correct implementation based on the source type + _iterate_spectra_fast_impl(f, data, data.source) +end + + diff --git a/src/MSI_src.jl b/src/MSI_src.jl new file mode 100644 index 0000000..c6f3ce0 --- /dev/null +++ b/src/MSI_src.jl @@ -0,0 +1,33 @@ +module MSI_src + +# Export the public API +export OpenMSIData, GetSpectrum, IterateSpectra, ImportMzmlFile, load_slices, plot_slices, plot_slice, get_total_spectrum, get_average_spectrum, LoadMzml, LoadSpectra + +# Include all source files directly into the main module +include("MSIData.jl") +include("ParserHelpers.jl") +include("mzML.jl") +include("imzML.jl") +include("MzmlConverter.jl") + + +# --- Main Entry Point --- # + +""" + OpenMSIData(filepath::String; cache_size=100) + +Opens a .mzML or .imzML file and prepares it for data access. + +This is the main entry point for the new data access API. +""" +function OpenMSIData(filepath::String; cache_size=100) + if endswith(lowercase(filepath), ".mzml") + return load_mzml_lazy(filepath, cache_size=cache_size) + elseif endswith(lowercase(filepath), ".imzml") + return load_imzml_lazy(filepath, cache_size=cache_size) + else + error("Unsupported file type: $filepath. Please provide a .mzML or .imzML file.") + end +end + +end # module MSI_src diff --git a/src/MzmlConverter.jl b/src/MzmlConverter.jl index 16a39ba..69e10be 100644 --- a/src/MzmlConverter.jl +++ b/src/MzmlConverter.jl @@ -1,16 +1,14 @@ -# src/MzmlConverter.jl - """ This file contains the workflow for converting .mzML files (with one spectrum per pixel) into a proper .imzML/.ibd file pair, using a separate synchronization file. -It replicates the functionality of the project's original R scripts that use MALDIquant. +It replicates the functionality of the original R scripts that use MALDIquant. """ -using CSV, DataFrames +using DataFrames, Printf, CSV + +# This file assumes that the main application file (e.g., app.jl) has already included +# the necessary source files: MSIData.jl, mzML.jl, imzML.jl -# Note: This file assumes that `LoadSpectra` from DataAccess.jl and the necessary -# structs/functions from ParserHelpers.jl, mzML.jl, and imzML.jl are available -# in the execution context (e.g., included in the main app.jl). # A struct to hold the processed pixel data before exporting struct ProcessedPixel @@ -19,11 +17,18 @@ struct ProcessedPixel intensity::Vector{Float32} end +# This struct will hold the metadata needed for the XML file +struct BinaryMetadata + mz_offset::UInt64 + mz_length::UInt64 + int_offset::UInt64 + int_length::UInt64 +end + """ GetMzmlScanTime(fileName::String) Parses a .mzML file to extract the scan start time for each spectrum. -This is a Julia implementation of the `GetMzmlScanTime` function from the R scripts. # Arguments * `fileName`: Path to the .mzML file. @@ -32,48 +37,75 @@ This is a Julia implementation of the `GetMzmlScanTime` function from the R scri - A `Matrix{Int64}` where each row is `[spectrum_index, time_in_milliseconds]`. """ function GetMzmlScanTime(fileName::String) - spec_count = 0 times = Tuple{Int64, Int64}[] + + # Pre-allocate based on an estimate to reduce re-allocations + try + file_size = filesize(fileName) + estimated_spectra = max(1000, file_size ÷ 10000) # Heuristic + sizehint!(times, estimated_spectra) + catch + # Ignore if filesize fails, sizehint! is just an optimization + end open(fileName, "r") do stream - file_content = read(stream, String) + state = :outside_spectrum + current_index = 0 + current_time = nothing - # First, find the total number of spectra to pre-allocate - spectrum_list_match = match(r" tag or count attribute.") - end - total_spectra = parse(Int, spectrum_list_match.captures[1]) - sizehint!(times, total_spectra) + for line in eachline(stream) + if state == :outside_spectrum + if occursin("]*>.*?<\/spectrum>", file_content, overlay=true) - spec_block = spectrum_match.match - # The R script appears to treat the 0-based mzML index as 1-based, so we add 1. - spec_index = parse(Int, spectrum_match.captures[1]) + 1 - - # Find the scan start time within the spectrum block - time_match = match(r"", line) + state = :outside_spectrum + if current_index > 0 && current_time !== nothing + push!(times, (current_index, current_time)) + end + end end end - - # Normalize times so the first scan is at t=0 - if !isempty(times) - sort!(times, by = x -> x[1]) # Ensure scans are sorted by index - first_time = times[1][2] - for i in eachindex(times) - times[i] = (times[i][1], times[i][2] - first_time) - end - end - end # open + end + + # Sort by spectrum index to ensure correct order before normalization + sort!(times, by = x -> x[1]) + # Normalize times relative to the first scan + if !isempty(times) + first_time = times[1][2] + for i in eachindex(times) + times[i] = (times[i][1], times[i][2] - first_time) + end + end + # Convert vector of tuples to a matrix + if isempty(times) + return Matrix{Int64}(undef, 0, 2) + end return permutedims(hcat(collect.(times)...)) end @@ -92,144 +124,491 @@ This is a Julia implementation of the `MatchAcquireTime` function from the R scr # Returns - A `Matrix{Int64}` where each row is `[pixel_index, pixel_time_ms, first_scan_index, last_scan_index]`. """ -function MatchAcquireTime(sync_file_path::String, scans::Matrix{Int64}) +function MatchAcquireTime(sync_file_path::String, scans::Matrix{Int64}; img_width::Int=0, img_height::Int=0) if !isfile(sync_file_path) error("Synchronization file not found: $sync_file_path") end - # Load xyz stage pixel delays from the sync file, skipping the first 2 header lines. pixel_df = CSV.read(sync_file_path, DataFrame, header=false, skipto=3) - pixel = Matrix(pixel_df) + pixel_matrix = Matrix(pixel_df) - # Normalize pixel times so the first is at t=0 - pixel[:, 2] .-= pixel[1, 2] + num_pixels_original = size(pixel_matrix, 1) + if num_pixels_original == 0 + return zeros(Int64, 0, 5) + end + + # Determine image dimensions and generate coordinates + current_img_width = img_width + current_img_height = img_height + + if current_img_width == 0 || current_img_height == 0 + if size(pixel_matrix, 2) >= 3 + @info "Sync file contains X, Y coordinates" + coordinates = convert(Matrix{Int}, pixel_matrix[:, 1:2]) + pixel_times = convert(Vector{Int64}, pixel_matrix[:, 3]) + current_img_width = maximum(coordinates[:, 1]) + current_img_height = maximum(coordinates[:, 2]) + + # Apply R's pixel truncation logic + num_pixels = (floor(Int, (num_pixels_original - 1) / current_img_width)) * current_img_width + if num_pixels <= 0 + error("Calculated num_pixels to process is zero or negative: $num_pixels") + end + + # Truncate arrays + final_coordinates = coordinates[1:num_pixels, :] + final_pixel_times = pixel_times[1:num_pixels] + + else + @info "Sync file contains Index and Time; generating coordinates" + pixel_times = convert(Vector{Int64}, pixel_matrix[:, 2]) + + # R's width detection logic + diffs = pixel_matrix[2:end, 1] .- pixel_matrix[1:end-1, 1] + width_indices = findall(x -> x != 1, diffs) + if !isempty(width_indices) + current_img_width = width_indices[1] + else + current_img_width = num_pixels_original + end + current_img_height = num_pixels_original ÷ current_img_width + + # R's pixel truncation logic + num_pixels = (floor(Int, (num_pixels_original - 1) / current_img_width)) * current_img_width + if num_pixels <= 0 + error("Calculated num_pixels to process is zero or negative: $num_pixels") + end + + # Generate coordinates for truncated pixels + final_coordinates = zeros(Int, num_pixels, 2) + final_pixel_times = zeros(Int64, num_pixels) + + for i in 1:num_pixels + idx = pixel_matrix[i, 1] + final_coordinates[i, 1] = ((idx - 1) % current_img_width) + 1 + final_coordinates[i, 2] = fld(idx - 1, current_img_width) + 1 + final_pixel_times[i] = pixel_times[i] + end + end + else + @info "Using provided dimensions: $(current_img_width)x$(current_img_height)" + pixel_times = convert(Vector{Int64}, pixel_matrix[:, 2]) + + # R's pixel truncation logic (FIXED: consistent formula) + num_pixels = (floor(Int, (num_pixels_original - 1) / current_img_width)) * current_img_width + if num_pixels <= 0 + error("Calculated num_pixels to process is zero or negative: $num_pixels") + end + + # Generate coordinates + final_coordinates = zeros(Int, num_pixels, 2) + final_pixel_times = zeros(Int64, num_pixels) + + for i in 1:num_pixels + idx = pixel_matrix[i, 1] + final_coordinates[i, 1] = ((idx - 1) % current_img_width) + 1 + final_coordinates[i, 2] = fld(idx - 1, current_img_width) + 1 + final_pixel_times[i] = pixel_times[i] + end + end + + # Normalize pixel times + if !isempty(final_pixel_times) + min_pixel_time = minimum(final_pixel_times) + final_pixel_times .-= min_pixel_time + end - num_pixels = size(pixel, 1) num_scans = size(scans, 1) - - index_matrix = zeros(Int64, num_pixels, 2) - - scan_row = 1 - for pixel_row in 1:(num_pixels - 1) - start_scan = scan_row - - # Find the scan that ends after the *next* pixel starts - while scan_row <= num_scans && scans[scan_row, 2] < pixel[pixel_row + 1, 2] - scan_row += 1 - end - - index_matrix[pixel_row, 1] = start_scan - index_matrix[pixel_row, 2] = scan_row - 1 - - # The next pixel's scans start from the one that crossed the boundary - scan_row = max(1, scan_row - 1) + if num_scans == 0 + return hcat(final_coordinates, final_pixel_times, zeros(Int64, num_pixels, 2)) end - # Assign scans for the last pixel - index_matrix[num_pixels, 1] = scan_row - index_matrix[num_pixels, 2] = num_scans + # FIXED R's time matching algorithm with bounds checking + first_idx = 1 + last_idx = 1 + index_matrix = zeros(Int, num_pixels, 2) - # Handle cases where some pixels have no scans by creating an empty range - for i in 1:num_pixels - if index_matrix[i, 2] < index_matrix[i, 1] - index_matrix[i, 2] = index_matrix[i, 1] - 1 + for i_pixel in 1:num_pixels + pixel_time = final_pixel_times[i_pixel] + + # Find the first scan that reaches or exceeds pixel time + while last_idx <= num_scans && scans[last_idx, 2] < pixel_time + last_idx += 1 + end + + # Ensure valid indices + if last_idx > num_scans + # No more scans available for remaining pixels + index_matrix[i_pixel:end, 1] .= num_scans + 1 # Invalid index + index_matrix[i_pixel:end, 2] .= num_scans # Invalid index + break + end + + # Assign scan indices with bounds checking + start_scan = max(1, first_idx) + end_scan = max(1, last_idx - 1) + + # Ensure start_scan <= end_scan + if start_scan > end_scan + start_scan = end_scan + end + + index_matrix[i_pixel, 1] = start_scan + index_matrix[i_pixel, 2] = end_scan + + # Update for next iteration (R's algorithm) + first_idx = last_idx - 1 + last_idx = first_idx + + # Ensure first_idx doesn't go below 1 + if first_idx < 1 + first_idx = 1 + last_idx = 1 end end - - return hcat(pixel, index_matrix) + + return hcat(final_coordinates, final_pixel_times, index_matrix) end -function RenderPixel(pixel_info, scans, spectra, scan_time_deltas, pixel_time_deltas) - first_scan = pixel_info[3] - last_scan = pixel_info[4] +function RenderPixel(pixel_info, scans, msi_data::MSIData, scan_time_deltas, pixel_time_deltas) + pixel_time = pixel_info[3] + first_scan = pixel_info[4] + last_scan = pixel_info[5] num_actions = last_scan - first_scan - # Use the m/z array of the first scan as the reference - mz_array = spectra[1, first_scan] + # Get reference m/z array from the first scan involved + mz_array, _ = GetSpectrum(msi_data, first_scan) new_intensity = zeros(Float32, length(mz_array)) + # SAFETY: Ensure we have valid scan indices + if first_scan < 1 || last_scan > size(scans, 1) || first_scan > last_scan + return (mz_array, new_intensity) # Return zero intensity for invalid ranges + end + if num_actions == 0 # Single scan contributes to the pixel - scale = pixel_time_deltas[pixel_info[1]] / scan_time_deltas[first_scan] - new_intensity .+= spectra[2, first_scan] .* scale + _, intensity = GetSpectrum(msi_data, first_scan) + # SAFETY: Ensure positive scaling + scale = max(pixel_time_deltas[first_scan] / scan_time_deltas[first_scan], 0.0f0) + new_intensity .= intensity .* scale elseif num_actions == 1 # Two partial scans contribute # First partial scan - scale1 = (scans[last_scan, 2] - pixel_info[2]) / scan_time_deltas[first_scan] - new_intensity .+= spectra[2, first_scan] .* scale1 + _, intensity1 = GetSpectrum(msi_data, first_scan) + scale1 = max((scans[last_scan, 2] - pixel_time) / scan_time_deltas[first_scan], 0.0f0) + new_intensity .+= intensity1 .* scale1 # Second partial scan - next_pixel_time = pixel_info[2] + pixel_time_deltas[pixel_info[1]] - scale2 = (next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan] - new_intensity .+= spectra[2, last_scan] .* scale2 + _, intensity2 = GetSpectrum(msi_data, last_scan) + next_pixel_time = pixel_time + pixel_time_deltas[first_scan] + scale2 = max((next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan], 0.0f0) + new_intensity .+= intensity2 .* scale2 elseif num_actions > 1 # Multiple scans contribute # First partial scan - scale1 = (scans[first_scan + 1, 2] - pixel_info[2]) / scan_time_deltas[first_scan] - new_intensity .+= spectra[2, first_scan] .* scale1 + _, intensity1 = GetSpectrum(msi_data, first_scan) + scale1 = max((scans[first_scan + 1, 2] - pixel_time) / scan_time_deltas[first_scan], 0.0f0) + new_intensity .+= intensity1 .* scale1 # Full scans in the middle for i in (first_scan + 1):(last_scan - 1) - new_intensity .+= spectra[2, i] + _, intensity_middle = GetSpectrum(msi_data, i) + new_intensity .+= intensity_middle end # Last partial scan - next_pixel_time = pixel_info[2] + pixel_time_deltas[pixel_info[1]] - scale2 = (next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan] - new_intensity .+= spectra[2, last_scan] .* scale2 + _, intensity2 = GetSpectrum(msi_data, last_scan) + next_pixel_time = pixel_time + pixel_time_deltas[first_scan] + scale2 = max((next_pixel_time - scans[last_scan, 2]) / scan_time_deltas[last_scan], 0.0f0) + new_intensity .+= intensity2 .* scale2 end + # FINAL SAFETY: Clamp any negative values to zero + new_intensity = max.(new_intensity, 0.0f0) + return (mz_array, new_intensity) end function ConvertMzmlToImzml(source_file::String, timing_matrix::Matrix{Int64}, scans::Matrix{Int64}) - # Get image width from timing info - width = findfirst(i -> timing_matrix[i, 1] - timing_matrix[i-1, 1] != 1, 2:size(timing_matrix, 1)) - height = size(timing_matrix, 1) ÷ width + if size(timing_matrix, 1) == 0 + return ProcessedPixel[], (0, 0) + end + + # Get image dimensions from timing_matrix (assuming columns 1 and 2 are X and Y) + width = maximum(timing_matrix[:, 1]) + height = maximum(timing_matrix[:, 2]) - # Load the full mzML data - spectra = LoadSpectra(source_file) + # Open the mzML file using the new memory-efficient API + msi_data = OpenMSIData(source_file) - # Pre-calculate time deltas - scan_time_deltas = diff(scans[:, 2]) - pixel_time_deltas = diff(timing_matrix[:, 2]) - # Append a final delta for the last element - push!(scan_time_deltas, scan_time_deltas[end]) - push!(pixel_time_deltas, pixel_time_deltas[end]) + # Pre-calculate time deltas robustly + scan_time_deltas = zeros(Int64, size(scans, 1)) + if size(scans, 1) > 1 + for i in 1:(size(scans, 1) - 1) + delta = scans[i+1, 2] - scans[i, 2] + scan_time_deltas[i] = max(1, delta) # Ensure delta is at least 1 + end + scan_time_deltas[end] = max(1, scan_time_deltas[end-1]) # Use previous delta for last scan, ensure at least 1 + end + + pixel_time_deltas = zeros(Int64, size(timing_matrix, 1)) + if size(timing_matrix, 1) > 1 + for i in 1:(size(timing_matrix, 1) - 1) + # Assumes time is in the 3rd column of the timing_matrix + delta = timing_matrix[i+1, 3] - timing_matrix[i, 3] + pixel_time_deltas[i] = max(1, delta) # Ensure delta is at least 1 + end + pixel_time_deltas[end] = max(1, pixel_time_deltas[end-1]) # Use previous delta for last pixel, ensure at least 1 + end processed_pixels = ProcessedPixel[] sizehint!(processed_pixels, size(timing_matrix, 1)) + empty_pixel_count = 0 + + # DEBUG: Scan coverage analysis + println("DEBUG: Scan coverage analysis:") + covered_pixels_count = 0 + for i in 1:size(timing_matrix, 1) + first_scan = timing_matrix[i, 4] + last_scan = timing_matrix[i, 5] + if first_scan <= last_scan && first_scan >= 1 && last_scan <= size(scans, 1) + covered_pixels_count += 1 + end + end + println(" Pixels with potential scans: $covered_pixels_count/$(size(timing_matrix, 1))") + println(" Total scans available: $(size(scans, 1))") for i in 1:size(timing_matrix, 1) pixel_info = timing_matrix[i, :] - first_scan = pixel_info[3] - last_scan = pixel_info[4] + x = pixel_info[1] + y = pixel_info[2] + first_scan = pixel_info[4] + last_scan = pixel_info[5] - # Skip if there are no scans for this pixel - (first_scan > last_scan) && continue + if first_scan > last_scan || first_scan < 1 || last_scan > size(scans, 1) + empty_pixel_count += 1 + push!(processed_pixels, ProcessedPixel((x, y), Float64[], Float32[])) + continue + end - # Calculate coordinates - x = ((pixel_info[1] - 1) % width) + 1 - y = fld(pixel_info[1] - 1, width) + 1 - - mz, intensity = RenderPixel(pixel_info, scans, spectra, scan_time_deltas, pixel_time_deltas) + mz, intensity = RenderPixel(pixel_info, scans, msi_data, scan_time_deltas, pixel_time_deltas) push!(processed_pixels, ProcessedPixel((x, y), mz, intensity)) end + @info "Found and created $empty_pixel_count empty pixels out of $(size(timing_matrix, 1)) total." return processed_pixels, (width, height) end function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims::Tuple{Int, Int}) - @warn "ExportImzml is not yet implemented. The processed pixel data has been generated but not saved to .imzML/.ibd files." - # TODO: Implement the logic to write the .imzML (XML) and .ibd (binary) files. - # 1. Open target_file.imzML and target_file.ibd for writing. - # 2. Write all m/z and intensity arrays sequentially to the .ibd file, tracking offsets and lengths. - # 3. Write the .imzML XML structure, including: - # - Boilerplate headers - # - , , - # - A with a for each pixel. - # - Each must contain cvParams for x/y coords and external data pointers to the .ibd file. - return false + ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") + + # Ensure we have valid pixels + if isempty(pixels) + @error "No pixels to export" + return false + end + + binary_meta = BinaryMetadata[] + sizehint!(binary_meta, length(pixels)) + + try + # Step 1: Write .ibd file with proper binary format + open(ibd_file, "w") do ibd_stream + # Write UUID as first 16 bytes (placeholder) + write(ibd_stream, zeros(UInt8, 16)) + + # Write in little-endian format (standard for imzML) + for pixel in pixels + # Write m/z array as 64-bit little-endian floats + mz_offset = position(ibd_stream) + for val in pixel.mz + write(ibd_stream, htol(Float64(val))) # Host to little-endian + end + mz_length = position(ibd_stream) - mz_offset + + # Write intensity array as 32-bit little-endian floats + int_offset = position(ibd_stream) + for val in pixel.intensity + write(ibd_stream, htol(Float32(val))) # Host to little-endian + end + int_length = position(ibd_stream) - int_offset + + push!(binary_meta, BinaryMetadata(mz_offset, mz_length, int_offset, int_length)) + end + end + + # Step 2: Write proper .imzML XML file + open(target_file, "w") do imzml_stream + # Use indexedmzML wrapper + write(imzml_stream, """ + + +""") + + # CV List + write(imzml_stream, """ + + + + +""") + + # File Description + write(imzml_stream, """ + + + + + + +""") + + # Referenceable Param Groups + write(imzml_stream, """ + + + + + + + + + + + + + +""") + + # Sample List + write(imzml_stream, """ + + + + +""") + + # Software List - OPEN SOURCE + write(imzml_stream, """ + + + + +""") + + # Scan Settings IMAGING + write(imzml_stream, """ + + + + + + + +""") + + # Instrument Configuration MALDI/TOF + write(imzml_stream, """ + + + + + + + + + + + + + + + +""") + + # Data Processing + write(imzml_stream, """ + + + + + + +""") + + # Run and Spectrum List + spectrum_offsets = UInt64[] + write(imzml_stream, """ + +""") + + # Write each spectrum + for (i, pixel) in enumerate(pixels) + meta = binary_meta[i] + x, y = pixel.coords + + spectrum_start = position(imzml_stream) + push!(spectrum_offsets, spectrum_start) + + write(imzml_stream, """ + + + + + + + + + + + + + + + + + + + + + + + + + +""") + end + + write(imzml_stream, """ + + +""") + # Index List + index_list_start = position(imzml_stream) + write(imzml_stream, """ + +""") + for (i, offset) in enumerate(spectrum_offsets) + write(imzml_stream, " $offset\n") + end + write(imzml_stream, """ + + $index_list_start + +""") + end + + println("Successfully created: $target_file") + println("Successfully created: $ibd_file") + return true + + catch e + @error "Failed to export imzML files" exception=(e, catch_backtrace()) + # Clean up partial files + isfile(ibd_file) && rm(ibd_file, force=true) + isfile(target_file) && rm(target_file, force=true) + return false + end end """ @@ -241,13 +620,15 @@ Main workflow function to convert a .mzML file to an .imzML file. * `source_file`: Path to the input .mzML file. * `sync_file`: Path to the synchronization text file. * `target_file`: Path for the output .imzML file (the .ibd will be named accordingly). +* `img_width`: width dimention for the creation of the x axis +* `img_height`: height dimention for the creation of the y axis """ -function ImportMzmlFile(source_file::String, sync_file::String, target_file::String) +function ImportMzmlFile(source_file::String, sync_file::String, target_file::String; img_width::Int=0, img_height::Int=0) println("Step 1: Getting scan times from .mzML file...") scans = GetMzmlScanTime(source_file) println("Step 2: Matching acquisition times...") - timing_matrix = MatchAcquireTime(sync_file, scans) + timing_matrix = MatchAcquireTime(sync_file, scans; img_width=img_width, img_height=img_height) println("Step 3: Converting spectra...") processed_pixels, (width, height) = ConvertMzmlToImzml(source_file, timing_matrix, scans) @@ -264,7 +645,7 @@ function ImportMzmlFile(source_file::String, sync_file::String, target_file::Str if success println("Conversion successful: $target_file") else - println("Conversion failed. Exporting is not yet implemented.") + println("Conversion failed.") end return success end diff --git a/src/ParserHelpers.jl b/src/ParserHelpers.jl index af53599..e0b21d6 100644 --- a/src/ParserHelpers.jl +++ b/src/ParserHelpers.jl @@ -51,7 +51,7 @@ Retrieves an attribute's value from an XML tag string. """ function get_attribute(source::AbstractString, tag::String = "([^=]+)") # Construct the regex pattern string - pattern_str = "\s" * tag * "=\"([^"]*)\"" + pattern_str = "\\s" * tag * "=\"([^\"]*)\"" regStr = Regex(pattern_str) return match(regStr, source) end @@ -107,16 +107,3 @@ mutable struct CVParams is_compressed::Bool axis_type::Symbol end - -function update_cv_params!(params::CVParams, acc::String) - if acc == "MS:1000514"; params.axis_type = :mz; - elseif acc == "MS:1000515"; params.axis_type = :intensity; - elseif acc == "MS:1000519"; params.format = Int32; - elseif acc == "MS:1000521"; params.format = Float32; - elseif acc == "MS:1000522"; params.format = Int64; - elseif acc == "MS:1000523"; params.format = Float64; - elseif acc == "MS:1000574"; params.is_compressed = true; - end -end - - diff --git a/src/imzML.jl b/src/imzML.jl index 2c87316..8b7ec9f 100644 --- a/src/imzML.jl +++ b/src/imzML.jl @@ -1,40 +1,4 @@ - -using Images, ImageFiltering, StatsBase, Statistics, CairoMakie, ColorSchemes, DataFrames, CSV, Printf, Dates -include("ParserHelpers.jl") - -# --- Lazy Loading Data Structures --- - -# Struct to hold metadata for a single spectrum, for lazy loading. -struct SpectrumMetadata - x::Int32 - y::Int32 - mz_offset::Int64 - intensity_offset::Int64 - mz_count::Int32 - intensity_count::Int32 -end - -# Struct to hold the parsed imzML metadata and the handle to the .ibd file. -mutable struct ImzMLData - ibd_handle::IO - mz_format::Type - intensity_format::Type - spectra_metadata::Vector{SpectrumMetadata} - width::Int - height::Int - mz_is_first::Bool # To know the order in the ibd file - - function ImzMLData(ibd_handle::IO, mz_format::Type, intensity_format::Type, spectra_metadata::Vector{SpectrumMetadata}, width::Int, height::Int, mz_is_first::Bool) - obj = new(ibd_handle, mz_format, intensity_format, spectra_metadata, width, height, mz_is_first) - finalizer(obj) do o - if isopen(o.ibd_handle) - close(o.ibd_handle) - end - end - return obj - end -end - +using Images, Statistics, CairoMakie, DataFrames, Printf, ColorSchemes # --- Extracted from imzML.jl --- @@ -43,7 +7,7 @@ This file provides a library for parsing `.imzML` and `.ibd` files in pure Julia It is intended to be included by a parent script. Core Functions: -- `load_imzml`: The main function that orchestrates the parsing. +- `load_imzml_lazy`: The main function that orchestrates the parsing. - Helper functions for reading XML metadata and binary spectral data. """ @@ -174,95 +138,115 @@ end """ - load_imzml(file_path::String) -Main function to parse an `.imzML`/.ibd file pair and prepare for lazy loading. -It parses the metadata from the `.imzML` file, including spectrum coordinates and -byte offsets for data in the `.ibd` file, but does not load the spectral data itself. + load_imzml_lazy(file_path::String; cache_size=100) -# Returns -- An `ImzMLData` struct containing the open `.ibd` file handle and all necessary metadata - for on-demand data loading. +Main function to parse an `.imzML`/.ibd file pair and prepare for lazy loading. +It now returns a unified MSIData object. """ -function load_imzml(file_path::String) +function load_imzml_lazy(file_path::String; cache_size=100) + println("DEBUG: Checking for .imzML file at $file_path") if !isfile(file_path) error("Provided path is not a file: $(file_path)") end - ibd_path = replace(file_path, ".imzML" => ".ibd") + ibd_path = replace(file_path, r"\.(imzML|mzML)"i => ".ibd") + println("DEBUG: Checking for .ibd file at $ibd_path") if !isfile(ibd_path) error("Corresponding .ibd file not found for: $(file_path)") end - stream = open(file_path) - hIbd = open(ibd_path) + println("DEBUG: Opening file streams for .imzML and .ibd") + stream = open(file_path, "r") + hIbd = open(ibd_path, "r") - # We don't use a try/finally block because we need to return the open hIbd handle. - # The finalizer on the ImzMLData struct will be responsible for closing it. + try + println("DEBUG: Configuring axes...") + axis = axes_config_img(stream) + println("DEBUG: Getting image dimensions...") + imgDim = get_img_dimensions(stream) + width, height, num_spectra = imgDim + println("DEBUG: Image dimensions: $(width)x$(height), $num_spectra spectra.") - axis = axes_config_img(stream) - imgDim = get_img_dimensions(stream) - width, height, num_spectra = imgDim + mz_config_idx = findfirst(a -> a.Axis == 1, axis) + int_config_idx = findfirst(a -> a.Axis == 2, axis) + mz_format = axis[mz_config_idx].Format + intensity_format = axis[int_config_idx].Format + println("DEBUG: m/z format: $mz_format, Intensity format: $intensity_format") - # Determine which format is for m/z and which for intensity from the referenceableParamGroups - mz_config_idx = findfirst(a -> a.Axis == 1, axis) - int_config_idx = findfirst(a -> a.Axis == 2, axis) - mz_format = axis[mz_config_idx].Format - intensity_format = axis[int_config_idx].Format + # --- NEW PARSING LOGIC based on the old, working code --- + println("DEBUG: Learning file structure from first spectrum...") + start_of_spectra_xml = position(stream) + attr = get_spectrum_attributes(stream, hIbd) + current_ibd_offset = position(hIbd) + seek(stream, start_of_spectra_xml) + println("DEBUG: Initial IBD offset: $current_ibd_offset") - # The original parsing logic is highly optimized and relies on fixed offsets within the XML structure. - # We reuse it to get the initial .ibd file offset and the structural attributes of the tags. - start_of_spectra_xml = position(stream) - attr = get_spectrum_attributes(stream, hIbd) - - # get_spectrum_attributes moves the hIbd pointer to the start of the first spectrum's binary data. - current_ibd_offset = position(hIbd) + spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra) + mz_is_first = attr[3] == 3 - # We need to re-scan the spectra list from the beginning to get metadata for each spectrum. - seek(stream, start_of_spectra_xml) + println("DEBUG: Parsing metadata for $num_spectra spectra using skip-based method...") + for k in 1:num_spectra + # Use skip values learned from the first spectrum, assuming all are identical. + skip(stream, attr[5]) # Skip to X coordinate value + val_tag_x = find_tag(stream, r"value=\"(\d+)\"") + x = parse(Int32, val_tag_x.captures[1]) - spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra) - - # attr[3] is 4 if the first binary data array is intensity, 3 if it's m/z. - mz_is_first = attr[3] == 3 + skip(stream, attr[6]) # Skip to Y coordinate value + val_tag_y = find_tag(stream, r"value=\"(\d+)\"") + y = parse(Int32, val_tag_y.captures[1]) - for k in 1:num_spectra - # These skips are based on the structure of the first spectrum tag, assuming all are identical. - skip(stream, attr[5]) # Skip to X coordinate value - val_tag = find_tag(stream, r"value=\"(\d+)\"") - x = parse(Int32, val_tag.captures[1]) + skip(stream, attr[7]) # Skip to array length value + val_tag_len = find_tag(stream, r"value=\"(\d+)\"") + nPoints = parse(Int32, val_tag_len.captures[1]) - skip(stream, attr[6]) # Skip to Y coordinate value - val_tag = find_tag(stream, r"value=\"(\d+)\"") - y = parse(Int32, val_tag.captures[1]) + mz_len_bytes = nPoints * sizeof(mz_format) + int_len_bytes = nPoints * sizeof(intensity_format) - skip(stream, attr[7]) # Skip to array length value - val_tag = find_tag(stream, r"value=\"(\d+)\"") - nPoints = parse(Int32, val_tag.captures[1]) + local mz_offset, int_offset + if mz_is_first + mz_offset = current_ibd_offset + int_offset = mz_offset + mz_len_bytes + else + int_offset = current_ibd_offset + mz_offset = int_offset + int_len_bytes + end - mz_len_bytes = nPoints * sizeof(mz_format) - int_len_bytes = nPoints * sizeof(intensity_format) - - local mz_offset, int_offset - if mz_is_first - mz_offset = current_ibd_offset - int_offset = mz_offset + mz_len_bytes - else - int_offset = current_ibd_offset - mz_offset = int_offset + int_len_bytes + # Create modern SpectrumAsset objects + mz_asset = SpectrumAsset(mz_format, false, mz_offset, nPoints, :mz) + int_asset = SpectrumAsset(intensity_format, false, int_offset, nPoints, :intensity) + + spectra_metadata[k] = SpectrumMetadata(x, y, "", mz_asset, int_asset) + + # Advance the offset for the next spectrum's data. + current_ibd_offset += mz_len_bytes + int_len_bytes + + skip(stream, attr[8]) # Skip to the end of the spectrum tag end + # --- END OF NEW PARSING LOGIC --- - spectra_metadata[k] = SpectrumMetadata(x, y, mz_offset, int_offset, nPoints, nPoints) + println("DEBUG: Metadata parsing complete.") - # Advance the offset for the next spectrum's data. - current_ibd_offset += mz_len_bytes + int_len_bytes - - skip(stream, attr[8]) # Skip to the end of the spectrum tag + # Build coordinate map for imzML files + println("DEBUG: Building coordinate map...") + coordinate_map = zeros(Int, width, height) + for (idx, meta) in enumerate(spectra_metadata) + if 1 <= meta.x <= width && 1 <= meta.y <= height + coordinate_map[meta.x, meta.y] = idx + end + end + println("DEBUG: Coordinate map built.") + + close(stream) + + source = ImzMLSource(hIbd, mz_format, intensity_format) + println("DEBUG: Creating MSIData object.") + return MSIData(source, spectra_metadata, (width, height), coordinate_map, cache_size) + + catch e + close(stream) + close(hIbd) + rethrow(e) end - - # Close the .imzML file stream, but leave the .ibd stream open for lazy reading. - close(stream) - - return ImzMLData(hIbd, mz_format, intensity_format, spectra_metadata, width, height, mz_is_first) end # --- End of content from imzML.jl --- @@ -310,9 +294,8 @@ end load_slices(folder, masses, tolerance) Loads image slices for multiple masses from all `.imzML` files in a directory. -This function is optimized to read the binary data file (`.ibd`) only once per file. -It iterates through each spectrum, reads its data, and finds all target masses -before proceeding to the next spectrum, minimizing disk I/O. +This function is now refactored to use the new MSIData architecture and its +caching capabilities. """ function load_slices(folder, masses, tolerance) files = filter(f -> endswith(f, ".imzML"), readdir(folder, join=true)) @@ -331,43 +314,27 @@ function load_slices(folder, masses, tolerance) push!(names, name) @info "Processing $(i)/$(n_files): $(name)" - imzML_data = @time load_imzml(file) # This is fast, it's fine. + # Load data using the new lazy loader, returning an MSIData object + msi_data = @time load_imzml_lazy(file) # Create empty images for all slices for the current file - current_file_slices = [zeros(Float64, imzML_data.width, imzML_data.height) for _ in 1:n_slices] - - hIbd = imzML_data.ibd_handle - mz_format = imzML_data.mz_format - int_format = imzML_data.intensity_format - - # Pre-allocate arrays to be reused for each spectrum - mz_array = Array{mz_format}(undef, 0) - intensity_array = Array{int_format}(undef, 0) - - # Iterate through each spectrum ONCE - for meta in imzML_data.spectra_metadata - # Resize arrays if the number of points in the spectrum has changed - if length(mz_array) != meta.mz_count - resize!(mz_array, meta.mz_count) - resize!(intensity_array, meta.intensity_count) - end - - # Read spectrum data ONCE from the .ibd file - seek(hIbd, meta.mz_offset) - read!(hIbd, mz_array) - seek(hIbd, meta.intensity_offset) - read!(hIbd, intensity_array) + width, height = msi_data.image_dims + current_file_slices = [zeros(Float64, width, height) for _ in 1:n_slices] + # Use the high-performance iterator to process all spectra + _iterate_spectra_fast(msi_data) do spec_idx, mz_array, intensity_array + meta = msi_data.spectra_metadata[spec_idx] + # Now, check for all masses of interest in this single spectrum for (j, mass) in enumerate(masses) intensity = find_mass(mz_array, intensity_array, mass, tolerance) if intensity > 0.0 - if 1 <= meta.x <= imzML_data.width && 1 <= meta.y <= imzML_data.height - current_file_slices[j][meta.x, meta.y] = intensity + if 1 <= meta.x <= width && 1 <= meta.y <= height + current_file_slices[j][meta.y, meta.x] = intensity end end end - end # end of spectra loop + end # end of fast iterator # Assign the generated images to the main list for j in 1:n_slices @@ -379,6 +346,70 @@ function load_slices(folder, masses, tolerance) return (img_list, names) end + +""" + plot_slice(msi_data::MSIData, mass::Float64, tolerance::Float64, output_dir::String; stage_name="slice", bins=256) + +Generates and saves a plot of a single image slice for a given m/z value. +This function closely imitates the logic of the original `GetSlice` but uses +the modern `MSIData` access patterns and robust peak finding. +""" +function plot_slice(msi_data::MSIData, mass::Real, tolerance::Real, output_dir::String; stage_name="slice_mz_$(mass)", bins=256) + + # 1. Create an empty image for the slice, with dimensions matching plotting expectations + width, height = msi_data.image_dims + slice_matrix = zeros(Float64, height, width) + + # 2. Iterate through each spectrum to build the slice + println("Generating slice for m/z $mass...") + _iterate_spectra_fast(msi_data) do spec_idx, mz_array, intensity_array + meta = msi_data.spectra_metadata[spec_idx] + + # Find the peak intensity using the modern, robust find_mass + intensity = find_mass(mz_array, intensity_array, mass, tolerance) + + if intensity > 0.0 + # Populate the matrix using (y, x) indexing + if 1 <= meta.x <= width && 1 <= meta.y <= height + slice_matrix[meta.y, meta.x] = intensity + end + end + end + println("Slice generation complete.") + + # 3. Plot the resulting slice matrix using CairoMakie + println("Plotting slice...") + + fig = Figure(size = (600, 500)) + ax = CairoMakie.Axis(fig[1, 1], + aspect=DataAspect(), + title=@sprintf("Slice for m/z: %.2f", mass) + ) + hidedecorations!(ax) + + # Use mass-specific bounds for colorrange, ensuring a valid range + min_val, max_val = extrema(slice_matrix) + if min_val == max_val + max_val = min_val + 1.0 # Ensure the color range has a non-zero width + end + hm = heatmap!(ax, slice_matrix, + colormap=cgrad(ColorSchemes.viridis, bins), + colorrange=(min_val, max_val) + ) + + Colorbar(fig[1, 2], hm, label="Intensity") + colgap!(fig.layout, 5) + + # 4. Save the plot + mkpath(output_dir) + filename = "$(stage_name).png" + save_path = joinpath(output_dir, filename) + save(save_path, fig) + @info "Saved slice plot to $save_path" + + return fig +end + # ============================================================================ # # @@ -647,53 +678,3 @@ function plot_slices(slices, names, masses, output_dir; stage_name, bins=256, dp end return fig end - -# ============================================================================= -# -# -# Main Workflow -# -# ============================================================================ - -function load_config(path) - config = Dict{String, Any}() - if !isfile(path) - @warn "Config file not found at $(path). Using default parameters." - # Define defaults here in case the file is missing - config["masses"] = [100] - config["tolerance"] = 0.1 - config["color_depth"] = 255 - config["outlier_prob"] = 0.98 - return config - end - - for line in eachline(path) - line = strip(line) - if isempty(line) || startswith(line, "#") - continue # Skip empty lines and comments - end - parts = split(line, '=', limit=2) - if length(parts) != 2 - @warn "Skipping malformed line in config: $(line)" - continue - end - key = strip(parts[1]) - value_str = strip(parts[2]) - - try - if key == "masses" - values = [parse(Float64, s) for s in split(value_str, ',')] - config[key] = values - elseif key == "tolerance" - config[key] = parse(Float64, value_str) - elseif key == "color_depth" - config[key] = parse(Int, value_str) - elseif key == "outlier_prob" - config[key] = parse(Float64, value_str) - end - catch e - @error "Could not parse value for key '$(key)': $(value_str)" - end - end - return config -end diff --git a/src/imzML_old.jl b/src/imzML_old.jl deleted file mode 100644 index dbb1480..0000000 --- a/src/imzML_old.jl +++ /dev/null @@ -1,246 +0,0 @@ -# include( "Common.jl" ); -# ******************************************************************* -# Load Spectra and return a matrix -# fileName: Full name path -# ******************************************************************* -""" - LoadImzml( fileName ) - -Load an imzML file as a matrix. Each column stores x-pixel position, - y-pixel position, x-axis data and y-axis data. - -# Arguments -* `fileName`: Full path name of the imzML file - -# Examples -```julia -# Load DESI MSI Carcinoma image data -spectra = LoadImzml( "80TopL, 50TopR, 70BottomL, 60BottomR-centroid.imzML" ) -size( spectra ) -(4, 18632) -``` -""" -function LoadImzml( fileName ) - - # Open file handles - if !isfile(fileName) - error("provided path is not a file") - end - - if endswith(fileName, ".imzML") - stream = open(fileName) - hIbd = open(replace(fileName, ".imzML" => ".ibd")) - else - stream = open(fileName * ".imzML") - hIbd = open(fileName * ".ibd") - end - - # Get axes types and image dimensions - axis = AxesConfigImg( stream ) - imgDim = GetImgDimensions( stream ) - format = [ axis[1].Format, axis[2].Format ] - - # Locate spectrum attributes - start = position( stream ) - attr = GetSpectrumAttributes( stream, hIbd ) - - # Load spectra - seek( stream, start ) - spectra = LoadImgData( stream, hIbd, attr, imgDim[3], format ) - - close( stream ) - close( hIbd ) - return spectra - -end - - -# ******************************************************************* -# Get axes value type -# ******************************************************************* -function AxesConfigImg( stream ) - - # Locate which axis is defined at first - tag = FindTag( stream, r"^\s*<(referenceableParamGroup )" ) - value = GetAttribute( tag.captures[1], "intensityArray" ) - order = 1 + ( value !== nothing ) - - # Read first axis configuration - axis = Array{ SpecDim, 1 }( undef, 2 ) - axis[ order ] = ConfigureSpecDim( stream ) - - # Read second axis configuration - FindTag( stream, r"^\s*<(referenceableParamGroup )" ) - axis[ xor( order,3 ) ] = ConfigureSpecDim( stream ) - return axis - -end - - -# ******************************************************************* -# Get vector's storage options and image dimensions -# ******************************************************************* -function GetImgDimensions( stream ) - - # Looks for "scanSettings" tag - # FindImgTag( stream, "scanSettings" ) - FindTag( stream, r"^\s*<(scanSettings )" ) - - # Initial values for dimension retrieve - n = 2 - dim = [ 0, 0, 0 ] - currLine = "" - matchInfo = RegexMatch - - while( n > 0 ) - - # Next field - currLine = readline( stream ) - matchInfo = match( r"^\s*<(cvParam)", currLine ) - index = length( matchInfo.captures[1] ) - matchInfo = GetAttribute( currLine[index:end], "accession" ) - - # Get axis identity - if( matchInfo.captures[1] == "IMS:1000042" # max X - || matchInfo.captures[1] == "IMS:1000043" ) # max Y - - # Read dimension's pixels - axis = matchInfo.captures[1][end] - '1' - index += matchInfo.offsets[1] + length( matchInfo.captures[1] ) - matchInfo = GetAttribute( currLine[index:end], "value" ) - dim[axis] = parse( Int32, matchInfo.captures[1] ) - n -= 1 - end - end - - # Load stored spectra counter - matchInfo = FindTag( stream, r"^\s*", line); break; end + if occursin("", line) + break + end if occursin("") - if binary_tag === nothing; error("Cannot find binary tag"); end + readuntil(stream, "") binary_offset = position(stream) - find_tag(stream, r"") # Position for next call + # Move stream to the end of the binary data array for the next iteration + readuntil(stream, "") - return SpectrumAsset(params.format, params.is_compressed, binary_offset, encoded_length, params.axis_type) + # Create SpectrumAsset directly from the variables + return SpectrumAsset(data_format, compression_flag, binary_offset, encoded_length, axis) end +# This function is updated to return the generic SpectrumMetadata struct function parse_spectrum_metadata(stream, offset::Int64) seek(stream, offset) - id_match = find_tag(stream, r"]*>(\d+)" + + # Actively search for offset tags, ignoring other lines until the end of the index is found. + while !eof(stream) line = readline(stream) - m = match(r"(\d+)", line) - if m !== nothing - push!(offsets, parse(Int64, m.captures[1])) - elseif occursin("", line) || occursin("", line) + + # First, check for the end condition. + if occursin("", line) || occursin("", line) break end + + # If it's not the end, see if it's an offset tag. + m = match(offset_regex, line) + if m !== nothing + push!(offsets, parse(Int64, m.captures[1])) + end + # If it's neither, ignore the line and continue the loop. end return offsets end -function load_mzml_lazy(file_path::String) - stream = open(file_path) +# This is the main lazy-loading function for mzML, now returning an MSIData object. +function load_mzml_lazy(file_path::String; cache_size::Int=100) + println("DEBUG: Opening file stream for $file_path") + stream = open(file_path, "r") try + println("DEBUG: Seeking to end of file.") seekend(stream) - skip(stream, -4096) # Read last 4KB + println("DEBUG: Skipping back to read footer.") + skip(stream, -4096) footer = read(stream, String) + println("DEBUG: Footer read successfully.") index_offset_match = match(r"(\d+)", footer) if index_offset_match === nothing + close(stream) error("Could not find . File may not be an indexed mzML.") end + println("DEBUG: Found indexListOffset.") index_offset = parse(Int64, index_offset_match.captures[1]) + println("DEBUG: Seeking to index list at offset $index_offset.") seek(stream, index_offset) + println("DEBUG: Searching for ''.") if find_tag(stream, r"(?\d+)", footer ) - - if index !== nothing - - # Jump to index table - seek( stream, parse( Int64, index.captures[1] ) ) - - # Find first "index" tag - index = FindTag( stream, r"(?\d+)<", String( raw ) ) - - # Exit condition - if index === nothing - break - end - - # Save index - used += 1 - unused -= 1 - entry[used] = parse( Int64, index.captures[1] ) - - # Grow list when needed - if unused == 0 - append!( entry, zeros( Int64, 8 ) ) - unused = 8 - end - - end - - return entry[1:used] - -end - - -# ******************************************************************* -# Read vector data -# stream: Source file stream -# ******************************************************************* -function ReadVector( stream, axis ) - - # Read base64 vector - base64Vec = FindTag( stream, r"([^<]+)<" ) - base64Vec = Vector{ UInt8 }( base64Vec.captures[1] ) - - # Fill a IO stream with decoded data - if axis.Packed == 1 - io = IOBuffer( Libz.inflate( Decode64( base64Vec ) ) ) - else - io = IOBuffer( Decode64( base64Vec ) ) - end - - # Convert IO stream content as dataType vector - nElem = Int32( io.size / sizeof( axis.Format ) ) - out = Array{ axis.Format }( undef, nElem ) - read!( io, out ) - - return out - -end - - -# ******************************************************************* -# Get the decoding options for the first vector in spectrum -# stream: Valid file stream -# lastTag: Last tag to skip i.e. "binaryDataArray" -# ******************************************************************* -function AxisConfig( stream ) - - offset = position( stream ) - skip = 0 - while true - - # Load line - raw = readline( stream ) - tag = match( r"^\s*<([^\s]+)", raw ) - start = length( tag.match ) + 1 - - # Is a "cvParam" tag? - if tag.captures[1] == "cvParam" - - # Get accesion field value - attribute = GetAttribute( raw[ start:end ], "accession" ) - - # Subtract file name length from skip field - if attribute.captures[1] != "MS:1000796" - continue - else - start += attribute.offset + length( attribute.match ) - value = GetAttribute( raw[ start:end ], "value" ) - skip += length( value.captures[1] ) - continue - end - - # Exit condition - elseif tag.captures[1] == "binaryDataArray" - value = GetAttribute( raw[ start:end ], "encodedLength" ) - offset = position( stream ) - offset - front = ConfigureSpecDim( stream ) - front.Skip += offset - length( value.captures[1] ) - skip - return ( front ) - - end - end -end - - -# ******************************************************************* -# Load Spectra and return a matrix -# stream: Source file stream -# ******************************************************************* -function LoadSpectra( stream, offset ) - - # Move file pointer towards first spectrum tag - seek( stream, offset[1] ) - - # Decodes axis information and skip counter - firstAxis = AxisConfig( stream ) - secondAxis = AxisConfig( stream ) - - # Allocates memory for spectra storage - specCount = length( offset ) - spectrum = Array{ Any }( undef, ( 2, specCount ) ) - index = 1 - order = 1 + ( firstAxis.Axis == 2 ) - - for i in 1:specCount - - # Load first vector - seek( stream, offset[ i ] ) - skip( stream, firstAxis.Skip ) - spectrum[ order, i ] = ReadVector( stream, firstAxis ) - - # Load second vector - skip( stream, secondAxis.Skip ) - spectrum[ xor( order,3 ), i ] = ReadVector( stream, secondAxis ) - end - - return( spectrum ) - -end - - -# ******************************************************************* -# Decoding base64 blocks -# ******************************************************************* - -# Decoding exchange table -base64dec = Vector{UInt8}( [ - 62, # (01) + -> 43d, 0x2B - 0, # (02) , - 0, # (03) - - 0, # (04) . - 63, # (05) / - 52, # (06) 0 - 53, # (07) 1 - 54, # (08) 2 - 55, # (09) 3 - 56, # (10) 4 - 57, # (11) 5 - 58, # (12) 6 - 59, # (13) 7 - 60, # (14) 8 - 61, # (15) 9 - 0, # (16) : - 0, # (17) ; - 0, # (18) < - 0, # (19) = - 0, # (20) > - 0, # (21) ? - 0, # (22) @ - 0, # (23) A - 1, # (24) B - 2, # (25) C - 3, # (26) D - 4, # (27) E - 5, # (28) F - 6, # (29) G - 7, # (40) H - 8, # (41) I - 9, # (42) J - 10, # (43) K - 11, # (44) L - 12, # (45) M - 13, # (46) N - 14, # (47) O - 15, # (48) P - 16, # (49) Q - 17, # (40) R - 18, # (41) S - 19, # (42) T - 20, # (43) U - 21, # (44) V - 22, # (45) W - 23, # (46) X - 24, # (47) Y - 25, # (48) Z - 0, # (49) [ - 0, # (50) \ - 0, # (51) ] - 0, # (52) ^ - 0, # (53) _ - 0, # (54) ` - 26, # (55) a - 27, # (56) b - 28, # (57) c - 29, # (58) d - 30, # (59) e - 31, # (60) f - 32, # (61) g - 33, # (62) h - 34, # (63) i - 35, # (64) j - 36, # (65) k - 37, # (66) l - 38, # (67) m - 39, # (68) n - 40, # (69) o - 41, # (70) p - 42, # (71) q - 43, # (72) r - 44, # (73) s - 45, # (74) t - 46, # (75) u - 47, # (76) v - 48, # (77) w - 49, # (78) x - 50, # (79) y - 51, # (80) z -] ) - -# base64 decoding -function DecodeTriplet( data, base64 ) - - # Character to binary equivalent - data = data .- 0x2A - data[1] = base64[ data[1] ] - data[2] = base64[ data[2] ] - data[3] = base64[ data[3] ] - data[4] = base64[ data[4] ] - - # Get Triplet - decode = Array{UInt8}( undef,3 ) - decode[1] = data[1] << 2 + data[2] >> 4 - decode[2] = data[2] << 4 + data[3] >> 2 - decode[3] = data[3] << 6 + data[4] - return decode - -end - - -function Decode64( data ) - - # Reserve uninitialized memory - block = divrem( sizeof( data ), 4 ) - decode = Array{UInt8}( undef, 3*block[1] ) - - # Recover binary from Chars - index = 1 - first = 1 - last = 3 - - for i in 1:block[1] - decode[first:last] = DecodeTriplet( data[index:index+3], base64dec ) - index += 4 - first += 3 - last += 3 - end - - # Adjust block size - if data[end-1] == 0x3D - decode = decode[1:end-2] - elseif data[end] == 0x3D - decode = decode[1:end-1] - end - - return decode - -end diff --git a/test/readme.md b/test/readme.md new file mode 100644 index 0000000..bea2736 --- /dev/null +++ b/test/readme.md @@ -0,0 +1,21 @@ +to run the tests: + +time julia --project=. test/run_tests.jl + +to enable the different test cases, there are boolean variables + +test1 = true +test2 = true +test3 = true + +set them to true to perform that specific test. + +test 1 consists in mzml validation and spectrum plotting + +test 2 consists in imzml conversion and validation + +test 3 consists in imzml validation and spectrum and slice plotting + +use the global variables: +TEST_MZML_FILE, SPECTRUM_TO_PLOT, CONVERSION_SOURCE_MZML, CONVERSION_SYNC_FILE, CONVERSION_TARGET_IMZML, TEST_IMZML_FILE, MZ_VALUE_FOR_SLICE, MZ_TOLERANCE, COORDS_TO_PLOT, RESULTS_DIR +to modify input that the test script will recieve. diff --git a/test/run_tests.jl b/test/run_tests.jl new file mode 100644 index 0000000..d96c39f --- /dev/null +++ b/test/run_tests.jl @@ -0,0 +1,299 @@ +# test/run_tests.jl + +# =================================================================== +# Test Environment for JuliaMSI Package +# =================================================================== +# This script validates the core functionality of the data processing +# workflows, including loading, converting, and visualizing mass +# spectrometry data. +# +# Instructions: +# 1. Fill in the placeholder paths in the "CONFIG" section below. +# 2. Run the script from the project's root directory: +# julia test/run_tests.jl +# 3. Check the `test/results/` folder for the output images. +# =================================================================== + +using Printf +using CairoMakie +import Pkg + +# --- Load the MSI_src Module --- +# Activate the project environment at the parent directory of this test script +Pkg.activate(joinpath(@__DIR__, "..")) +using MSI_src + + +# =================================================================== +# CONFIG: PLEASE FILL IN YOUR FILE PATHS HERE +# =================================================================== + +# --- Test Case 1: Standard .mzML file --- +# A regular, non-imaging mzML file. +# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/mzML/T9_A1.mzML" +# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" +const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" +const SPECTRUM_TO_PLOT = 1 # Which spectrum to plot from the file + +# --- Test Case 2: .mzML + Sync File for Conversion --- +# The special .mzML file with one spectrum per pixel. +const CONVERSION_SOURCE_MZML = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" +# The corresponding synchronization text file. +const CONVERSION_SYNC_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.txt" + +# const CONVERSION_SOURCE_MZML = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.mzML" +# const CONVERSION_SYNC_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.txt" + +# The desired output path for the new .imzML file. +const CONVERSION_TARGET_IMZML = "test/results/converted_mzml.imzML" + +# --- Test Case 3: Standard .imzML file --- +# An existing imzML file (can be the one generated from Case 2). +const TEST_IMZML_FILE = CONVERSION_TARGET_IMZML # The output from case 2 +# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML" +# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.imzML" +# The m/z value to use for creating an image slice. +const MZ_VALUE_FOR_SLICE = 309.06 # BF +# const MZ_VALUE_FOR_SLICE = 896.0 # HR2MSI +# const MZ_VALUE_FOR_SLICE = 76.03 # I PS +# const MZ_TOLERANCE = 0.1 +const MZ_TOLERANCE = 1 + +# Coordinates to plot a specific spectrum from imzML +const COORDS_TO_PLOT = (50, 50) # Example coordinates (X, Y) + +# --- Output Directory --- +const RESULTS_DIR = "test/results" + +test1 = true +test2 = true +test3 = true + +# =================================================================== +# DATA VALIDATION UTILITY +# =================================================================== + +""" + validate_msi_data(filepath::String) + +Performs a series of checks on a .mzML or .imzML file using the MSIData API. +""" +function validate_msi_data(filepath::String) + println("" * "-"^10 * " Running validation for $filepath " * "-"^10) + if !isfile(filepath) + println("SKIPPED VALIDATION: File not found: $filepath") + return false + end + + try + # 1. Basic structure validation + println("Opening file with OpenMSIData...") + msi_data = @time OpenMSIData(filepath) + + # 2. Compare spectrum counts + num_spectra = length(msi_data.spectra_metadata) + println("Found $num_spectra spectra") + @assert num_spectra > 0 "No spectra found in file." + + # 3. Test random access + println("Testing random access to spectra...") + test_indices = unique([1, max(1, num_spectra ÷ 2), num_spectra]) + println("Testing indices: $test_indices") + for idx in test_indices + print("Fetching spectrum #$idx... ") + mz, intensity = @time GetSpectrum(msi_data, idx) + @assert length(mz) == length(intensity) "Spectrum $idx: mz/intensity length mismatch. Got $(length(mz)) mz values and $(length(intensity)) intensity values." + println("OK, $(length(mz)) points.") + end + + # 4. Test iteration + println("Testing iteration over all spectra...") + count = 0 + iter_time = @elapsed for (idx, (mz, intensity)) in IterateSpectra(msi_data) + count += 1 + # Basic data validation + @assert all(isfinite, mz) "Non-finite mz values in spectrum $idx" + @assert all(>=(0), intensity) "Negative intensities in spectrum $idx" + end + println("Iterated over $count spectra in $iter_time seconds.") + @assert count == num_spectra "Iteration count mismatch: expected $num_spectra, got $count." + + println("VALIDATION SUCCESSFUL for $filepath") + return true + + catch e + println("VALIDATION FAILED for $filepath.") + showerror(stdout, e, catch_backtrace()) + println() + return false + end +end + + +# =================================================================== +# TEST RUNNER +# =================================================================== + +function run_test() + println("Starting MSI_src Test Suite...") + + # --- Test Case 1: Process a standard .mzML file --- + println("" * "="^20 * " Test Case 1: Processing .mzML " * "="^20) + if test1 == true + # Run new, stronger validation + validate_msi_data(TEST_MZML_FILE) + + # Also run original plotting test to ensure visualization still works + if isfile(TEST_MZML_FILE) + try + # Get the msi data from the mzml + println("Plotting a sample spectrum from $TEST_MZML_FILE...") + msi_data = OpenMSIData(TEST_MZML_FILE) + mz, intensity = GetSpectrum(msi_data, SPECTRUM_TO_PLOT) + + fig = Figure(size = (800, 600)) + ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Intensity", title="Spectrum #$SPECTRUM_TO_PLOT from $(basename(TEST_MZML_FILE))") + lines!(ax, mz, intensity) + + output_path = joinpath(RESULTS_DIR, "test_mzml_spectrum.png") + save(output_path, fig) + println("SUCCESS: Spectrum plot saved to $output_path") + # Get the summed spectrum data + mz, intensity = get_total_spectrum(msi_data) + + # Plot the data + println("Plotting total spectrum...") + fig = Figure(size = (800, 600)) + ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Total Intensity", title="Total Spectrum from $(basename(TEST_MZML_FILE))") + lines!(ax, mz, intensity) + + # Saving the output + output_path = joinpath(RESULTS_DIR, "test_mzml_total_spectrum.png") + save(output_path, fig) + println("SUCCESS: Total spectrum plot saved to $output_path") + + # Get the averaged spectrum data + mz, intensity = get_average_spectrum(msi_data) + + # Plot the data + println("Plotting averaged spectrum...") + fig = Figure(size = (800, 600)) + ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Average Intensity", title="Average Spectrum from $(basename(TEST_MZML_FILE))") + lines!(ax, mz, intensity) + + # Saving the output + output_path = joinpath(RESULTS_DIR, "test_mzml_average_spectrum.png") + save(output_path, fig) + println("SUCCESS: Total spectrum plot saved to $output_path") + catch e + println("ERROR during plotting in Test Case 1: $e") + end + end + else + println("SKIPPED Test Case 1.") + end + + # --- Test Case 2: Convert .mzML + .txt to .imzML --- + println("" * "="^20 * " Test Case 2: Converting to .imzML " * "="^20) + if isfile(CONVERSION_SOURCE_MZML) && isfile(CONVERSION_SYNC_FILE) && test2 == true + try + println("Running conversion process (with profiling)...") + success = @time ImportMzmlFile(CONVERSION_SOURCE_MZML, CONVERSION_SYNC_FILE, CONVERSION_TARGET_IMZML) + if success + println("SUCCESS: Conversion process completed.") + # Validate the newly created imzML file + validate_msi_data(CONVERSION_TARGET_IMZML) + else + println("FAILURE: Conversion process failed.") + end + catch e + println("ERROR in Test Case 2: $e") + end + else + println("SKIPPED: Files not found for Test Case 2.") + println(" - mzML: $CONVERSION_SOURCE_MZML") + println(" - Sync: $CONVERSION_SYNC_FILE") + end + + # --- Test Case 3: Process an existing .imzML file --- + println(" +" * "="^20 * " Test Case 3: Processing .imzML " * "="^20) + if test3 == true + # Run new, stronger validation for imzML + validate_msi_data(TEST_IMZML_FILE) + + # Also run tests for plotting spectrum and image slice + if isfile(TEST_IMZML_FILE) + # Add spectrum plotting for imzML to match Test Case 1 + try + # Get the msi data from the imzml + println("Plotting a sample spectrum from $TEST_IMZML_FILE...") + msi_data = OpenMSIData(TEST_IMZML_FILE) + x_coord, y_coord = COORDS_TO_PLOT + + # Get the x y coordinate spectrum data + mz, intensity = GetSpectrum(msi_data, x_coord, y_coord) + + # Plot the data + fig = Figure(size = (800, 600)) + ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Intensity", title="Spectrum at ($x_coord, $y_coord) from $(basename(TEST_IMZML_FILE))") + lines!(ax, mz, intensity) + + # Saving the output + output_path = joinpath(RESULTS_DIR, "test_imzml_spectrum.png") + save(output_path, fig) + println("SUCCESS: Spectrum plot saved to $output_path") + + # Get the summed spectrum data + mz, intensity = get_total_spectrum(msi_data) + + # Plot the data + println("Plotting total spectrum...") + fig = Figure(size = (800, 600)) + ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Total Intensity", title="Total Spectrum from $(basename(TEST_IMZML_FILE))") + lines!(ax, mz, intensity) + + # Saving the output + output_path = joinpath(RESULTS_DIR, "test_imzml_total_spectrum.png") + save(output_path, fig) + println("SUCCESS: Total spectrum plot saved to $output_path") + + # Get the averaged spectrum data + mz, intensity = get_average_spectrum(msi_data) + + # Plot the data + println("Plotting averaged spectrum...") + fig = Figure(size = (800, 600)) + ax = Axis(fig[1, 1], xlabel="m/z", ylabel="Average Intensity", title="Average Spectrum from $(basename(TEST_IMZML_FILE))") + lines!(ax, mz, intensity) + + # Saving the output + output_path = joinpath(RESULTS_DIR, "test_imzml_average_spectrum.png") + save(output_path, fig) + println("SUCCESS: Total spectrum plot saved to $output_path") + catch e + println("ERROR during spectrum plotting in Test Case 3: $e") + end + + # Test the plot_slice function + try + println("Testing plot_slice function on $TEST_IMZML_FILE...") + msi_data = OpenMSIData(TEST_IMZML_FILE) + plot_slice(msi_data, MZ_VALUE_FOR_SLICE, MZ_TOLERANCE, RESULTS_DIR, stage_name="test_imzml_single_slice") + # The success message is now inside plot_slice + catch e + println("ERROR during plot_slice test in Test Case 3: $e") + end + end + else + println("SKIPPED Test Case 3.") + end + + println("Tests for all 3 cases is finished.") + +end + +# --- Execute --- +# Ensure the results directory exists +mkpath(RESULTS_DIR) +@time run_test() \ No newline at end of file From 22c8d9637b0ac0cfb20f07b28c62e1aa584e85a5 Mon Sep 17 00:00:00 2001 From: Pixelguy14 Date: Thu, 2 Oct 2025 13:28:15 -0600 Subject: [PATCH 03/19] integrated MSI_src API into the UI, fixed interface elements, made some processes asynchronous, modified spectrum plots, added sum spectrum, removed use of obsolete julia_mzML_imzML library --- Manifest.toml | 2 +- Project.toml | 2 + app.jl | 805 +++++++++++++++++++++--------------------- app.jl.html | 5 + julia_imzML_visual.jl | 248 ++++++------- src/MSIData.jl | 97 ++++- src/MzmlConverter.jl | 304 +++++++++------- src/imzML.jl | 214 ++++++++++- test/readme.md | 83 ++++- test/run_tests.jl | 24 +- 10 files changed, 1076 insertions(+), 708 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index ff5e2fb..e1ae255 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.11.7" manifest_format = "2.0" -project_hash = "896c16630591d2732da8e3315c4b64c618fd5ebb" +project_hash = "0f3959248d174e05d23b9e9267b00c6d2ead256d" [[deps.ATK_jll]] deps = ["Artifacts", "Glib_jll", "JLLWrappers", "Libdl"] diff --git a/Project.toml b/Project.toml index 653695f..0db27f1 100644 --- a/Project.toml +++ b/Project.toml @@ -19,6 +19,8 @@ NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4" NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85" PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" StipplePlotly = "ec984513-233d-481d-95b0-a3b58b97af2b" julia_mzML_imzML = "38eb50d3-2fb6-4afa-992a-964ed8562ed9" diff --git a/app.jl b/app.jl index b0fee09..b836bce 100644 --- a/app.jl +++ b/app.jl @@ -6,7 +6,8 @@ using Libz using PlotlyBase using CairoMakie using Colors -using julia_mzML_imzML +# using julia_mzML_imzML +using MSI_src # Import the new MSIData library using Statistics using NaturalSort using Images @@ -14,6 +15,10 @@ using LinearAlgebra using NativeFileDialog # Opens the file explorer depending on the OS using StipplePlotly using Base.Filesystem: mv # To rename files in the system + +# Bring MSIData into App module's scope +using .MSI_src: MSIData, OpenMSIData, GetSpectrum, IterateSpectra, ImzMLSource, _iterate_spectra_fast, MzMLSource, find_mass, ViridisPalette, get_mz_slice, quantize_intensity, save_bitmap, median_filter, save_bitmap, downsample_spectrum, TrIQ + include("./julia_imzML_visual.jl") @genietools @@ -57,6 +62,7 @@ include("./julia_imzML_visual.jl") @in compareBtn=false # To open dialog @in createMeanPlot=false # To generate mean spectrum plot @in createXYPlot=false # To generate an spectrum plot according to the xy values inputed + @in createSumPlot=false # To generate a sum of all the spectrum plots @in image3dPlot=false # To generate 3d plot based on current image @in triq3dPlot=false # To generate 3d plot based on current triq image @in imageCPlot=false # To generate contour plots of current image @@ -110,10 +116,12 @@ include("./julia_imzML_visual.jl") @out msgimgComp="" @out msgtriqComp="" + # Centralized MSIData object + @out msi_data::Union{MSIData, Nothing} = nothing + # Saves the route where imzML and mzML files are located @out full_route="" - @out full_routeMz="" - @out full_routeMz2="" + # For the creation of images with a more specific mass charge @out text_nmass="" @@ -145,10 +153,12 @@ include("./julia_imzML_visual.jl") layoutImg=PlotlyBase.Layout( xaxis=PlotlyBase.attr( visible=false, - scaleanchor="y" + scaleanchor="y", + range=[0, 0] ), yaxis=PlotlyBase.attr( - visible=false + visible=false, + range=[0, 0] ), margin=attr(l=0,r=0,t=0,b=0,pad=0) ) @@ -168,10 +178,10 @@ include("./julia_imzML_visual.jl") # Interface Plot Spectrum layoutSpectra=PlotlyBase.Layout( title="Spectrum plot", + hovermode="closest", xaxis=PlotlyBase.attr( title="m/z", - showgrid=true, - tickformat = ".3g" + showgrid=true ), yaxis=PlotlyBase.attr( title="Intensity", @@ -249,319 +259,305 @@ include("./julia_imzML_visual.jl") # Reactive handlers watch a variable and execute a block of code when its value changes # The onbutton handler will set the variable to false after the block is executed - @onbutton btnSearch begin - full_route=pick_file(; filterlist="imzML,imzml,mzML,mzml") - msg="" - if full_route != "" - if endswith(full_route, "imzml") - alt_route=replace(full_route, r"\.[^.]*$" => ".imzML") - mv(full_route, alt_route) - full_route=alt_route - # to detect if there's an mzml named wrong - alt_routeMz=replace(full_route, r"\.[^.]*$" => ".mzml") - if isfile(alt_routeMz) - alt_routeMz2=replace(alt_routeMz, r"\.[^.]*$" => ".mzML") - mv(alt_routeMz, alt_routeMz2) - alt_routeMz=alt_routeMz2 - end - end - if endswith(full_route, "mzml") - alt_route=replace(full_route, r"\.[^.]*$" => ".mzML") - mv(full_route, alt_route) - full_route = alt_route - # to detect if there's an imzml named wrong - alt_routeMz=replace(full_route, r"\.[^.]*$" => ".imzml") - if isfile(alt_routeMz) - alt_routeMz2=replace(alt_routeMz, r"\.[^.]*$" => ".imzML") - mv(alt_routeMz, alt_routeMz2) - alt_routeMz=alt_routeMz2 - end - end + @onbutton btnSearch @time begin + # This part is synchronous and blocking, which is unavoidable + picked_route = pick_file(; filterlist="imzML,imzml,mzML,mzml") + + if isempty(picked_route) + msg = "No file selected." + warning_msg = true + return end - if full_route=="" - msg="No file selected" - warning_msg=true - btnStartDisable=true - btnSpectraDisable=true - SpectraEnabled=false - else - if endswith(full_route, "imzML") # Case if the file loaded is imzML - btnStartDisable=false - btnPlotDisable=false - # Splitting the route with regex from imzml to mzml so the plotting can work - full_routeMz=replace(full_route, r"\.[^.]*$" => ".mzML") - if isfile(full_routeMz) - # Start the mean spectrum creation on loading - progressSpectraPlot=true - btnPlotDisable=true - btnStartDisable=true - msg="Loading mean spectrum plot..." - sTime=time() - plotdata, plotlayout, xSpectraMz, ySpectraMz=meanSpectrumPlot(full_routeMz) - selectedTab="tab2" - progressSpectraPlot=false - btnPlotDisable=false - # We enable coord search and spectra plot creation, also re-enable main process - btnSpectraDisable=false - SpectraEnabled=true - btnStartDisable=false - - fTime=time() - eTime=round(fTime-sTime,digits=3) - msg="Plot loaded in $(eTime) seconds" + # UI updates immediately + progress = true + msg = "Opening file: $(basename(picked_route))..." + + @async begin + try + # --- Normalize file extension and path --- + if endswith(picked_route, "imzml") + full_route = replace(picked_route, r"\.imzml$"i => ".imzML") + mv(picked_route, full_route, force=true) + elseif endswith(picked_route, "mzml") + full_route = replace(picked_route, r"\.mzml$"i => ".mzML") + mv(picked_route, full_route, force=true) else - # If there's no mzML file, we deny access again - btnSpectraDisable=true - SpectraEnabled=false + full_route = picked_route end - else # Case if the file loaded is mzML - full_routeMz=full_route - btnSpectraDisable=false - SpectraEnabled=true - if isfile(full_routeMz) - # Start the mean spectrum creation on loading - progressSpectraPlot=true - btnPlotDisable=true - btnStartDisable=true - msg="Loading mean spectrum plot..." - sTime=time() - plotdata, plotlayout, xSpectraMz, ySpectraMz=meanSpectrumPlot(full_routeMz) - selectedTab="tab2" - progressSpectraPlot=false - btnPlotDisable=false - # We enable coord search and spectra plot creation - btnSpectraDisable=false - SpectraEnabled=true + # --- Load data using the new MSIData library --- + sTime = time() + msi_data = OpenMSIData(full_route) + + w, h = msi_data.image_dims + imgWidth, imgHeight = w > 0 ? (w, h) : (500, 500) - fTime=time() - eTime=round(fTime-sTime,digits=3) - msg="Plot loaded in $(eTime) seconds" - end - # Splitting the route the same way - full_route=replace(full_route, r"\.[^.]*$" => ".imzML") - if isfile(full_route) - btnStartDisable=false - else - btnStartDisable=true - full_route=full_routeMz - end + fTime = time() + eTime = round(fTime - sTime, digits=3) + msg = "File loaded in $(eTime) seconds. Calculating total spectrum..." + + # Enable UI controls + btnStartDisable = !(msi_data.source isa ImzMLSource) + btnPlotDisable = false + btnSpectraDisable = false + SpectraEnabled = true + + # --- Automatically generate and display the sum spectrum plot --- + # This is still part of the same async task + progressSpectraPlot = true + sTime = time() + + plotdata, plotlayout, xSpectraMz, ySpectraMz = sumSpectrumPlot(msi_data) + + selectedTab = "tab2" + fTime = time() + eTime = round(fTime - sTime, digits=3) + msg = "Total spectrum plot loaded in $(eTime) seconds." + + catch e + msi_data = nothing + msg = "Error loading file: $e" + warning_msg = true + btnStartDisable = true + btnSpectraDisable = true + SpectraEnabled = false + @error "File loading failed" exception=(e, catch_backtrace()) + finally + # This now correctly runs after everything is finished + progress = false + progressSpectraPlot = false end - xCoord=0 - yCoord=0 end end - @onbutton mainProcess begin - progress=true # Start progress button animation - btnStartDisable=true # We disable the button to avoid multiple requests - btnPlotDisable=true - btnSpectraDisable=true - text_nmass=replace(string(Nmass), "." => "_") - sTime=time() - if isfile(full_route) && Nmass > 0 && Tol > 0 && Tol <=1 && colorLevel > 1 && colorLevel < 257 - msg="File exists, Nmass=$(Nmass) Tol=$(Tol). Loading file will begin, please be patient." + @onbutton mainProcess @time begin + # UI updates immediately + progress = true + btnStartDisable = true + btnPlotDisable = true + btnSpectraDisable = true + + @async begin try - spectra=LoadImzml(full_route) - msg="File loaded. Creating spectra with the specific mass and tolerance, please be patient." - slice=GetMzSliceJl(spectra,Nmass,Tol) - fig=CairoMakie.Figure(size=(150, 250)) # Container - # Append a query string to force the image to refresh - timestamp=string(time_ns()) - if triqEnabled # If we have TrIQ - if triqProb < 0.8 || triqProb > 1 - msg="Incorrect TrIQ values, please adjust accordingly and try again." - warning_msg=true - else - image_path=joinpath("./public", "TrIQ_$(text_nmass).bmp") - valid_slice=false - while Tol <= 1.0 && !valid_slice - try - slice=GetMzSliceJl(spectra, Nmass, Tol) - sliceTriq=TrIQ(slice, colorLevel, triqProb) - if MFilterEnabled # If the Median filter is ON - sliceTriq=medianFilterjl(sliceTriq) - end - valid_slice=true - catch e - msg="Warning: insufficient tolerance, inputs modified to allow the creation of an image regardless=$Tol: $e" - Tol += 0.1 - end - end - sliceTriq=reverse(sliceTriq, dims=2) - SaveBitmapCl(joinpath("public", "TrIQ_$(text_nmass).bmp"),sliceTriq,ViridisPalette) - # Use timestamp to refresh image interface container - imgIntT="/TrIQ_$(text_nmass).bmp?t=$(timestamp)" - plotdataImgT, plotlayoutImgT, imgWidth, imgHeight=loadImgPlot(imgIntT) - # Get current image - current_triq="TrIQ_$(text_nmass).bmp" - msgtriq="TrIQ image with the Nmass of $(replace(text_nmass, "_" => "."))" - # Create colorbar - bound=julia_mzML_imzML.GetOutlierThres(slice, triqProb) - levels=range(bound[1],stop=bound[2], length=8) - levels=vcat(levels, 2*levels[end]-levels[end-1]) - Colorbar(fig[1, 1], colormap=cgrad(:viridis, colorLevel, categorical=true), limits=(0, bound[2]),ticks=levels,tickformat=log_tick_formatter, label="Intensity", size=25) - save("public/colorbar_TrIQ_$(text_nmass).png", fig) - colorbarT="/colorbar_TrIQ_$(text_nmass).png?t=$(timestamp)" - # Get current colorbar - current_col_triq="colorbar_TrIQ_$(text_nmass).png" - # We update the directory to include the new placed images. - triq_bmp=sort(filter(filename -> startswith(filename, "TrIQ_") && endswith(filename, ".bmp"), readdir("public")),lt=natural) - col_triq_png=sort(filter(filename -> startswith(filename, "colorbar_TrIQ_") && endswith(filename, ".png"), readdir("public")),lt=natural) - fTime=time() - eTime=round(fTime-sTime,digits=3) - msg="The file has been created in $(eTime) seconds successfully inside the 'public' folder of the app" - selectedTab="tab1" - end - else # If we don't use TrIQ - image_path=joinpath("./public", "MSI_$(text_nmass).bmp") + text_nmass = replace(string(Nmass), "." => "_") + sTime = time() + + if msi_data === nothing || !(msi_data.source isa ImzMLSource) + msg = "No .imzML file loaded or selected file is not an .imzML. Please select a valid file." + warning_msg = true + elseif Nmass > 0 && Tol > 0 && Tol <= 1 && colorLevel > 1 && colorLevel < 257 + msg = "Creating image for Nmass=$(Nmass) Tol=$(Tol). Please be patient." try - sliceQuant=IntQuantCl(slice,Int(colorLevel-1)) - if MFilterEnabled # If the Median filter is ON - sliceQuant=medianFilterjl(sliceQuant) + # Use the new get_mz_slice with the centralized MSIData object + slice = get_mz_slice(msi_data, Nmass, Tol) + fig = CairoMakie.Figure(size=(150, 250)) # Container + timestamp = string(time_ns()) + + if triqEnabled # If we have TrIQ + if triqProb < 0.8 || triqProb > 1 + msg = "Incorrect TrIQ values, please adjust accordingly and try again." + warning_msg = true + else + sliceTriq = TrIQ(slice, colorLevel, triqProb) + if MFilterEnabled + sliceTriq = round.(UInt8, median_filter(sliceTriq)) + end + sliceTriq = reverse(sliceTriq, dims=2) + save_bitmap(joinpath("public", "TrIQ_$(text_nmass).bmp"), sliceTriq, ViridisPalette) + + imgIntT = "/TrIQ_$(text_nmass).bmp?t=$(timestamp)" + plotdataImgT, plotlayoutImgT, imgWidth, imgHeight = loadImgPlot(imgIntT) + current_triq = "TrIQ_$(text_nmass).bmp" + msgtriq = "TrIQ image with the Nmass of $(replace(text_nmass, "_" => "."))" + + bound = MSI_src.get_outlier_thres(slice, triqProb) + levels = range(bound[1], stop=bound[2], length=8) + levels = vcat(levels, 2 * levels[end] - levels[end-1]) + Colorbar(fig[1, 1], colormap=cgrad(:viridis, colorLevel, categorical=true), limits=(0, bound[2]), ticks=levels, tickformat=log_tick_formatter, label="Intensity", size=25) + save("public/colorbar_TrIQ_$(text_nmass).png", fig) + colorbarT = "/colorbar_TrIQ_$(text_nmass).png?t=$(timestamp)" + current_col_triq = "colorbar_TrIQ_$(text_nmass).png" + + triq_bmp = sort(filter(filename -> startswith(filename, "TrIQ_") && endswith(filename, ".bmp"), readdir("public")), lt=natural) + col_triq_png = sort(filter(filename -> startswith(filename, "colorbar_TrIQ_") && endswith(filename, ".png"), readdir("public")), lt=natural) + + fTime = time() + eTime = round(fTime - sTime, digits=3) + msg = "The TrIQ image has been created in $(eTime) seconds successfully inside the 'public' folder of the app" + selectedTab = "tab1" + end + else # If we don't use TrIQ + sliceQuant = quantize_intensity(slice, colorLevel) + if MFilterEnabled + sliceQuant = round.(UInt8, median_filter(sliceQuant)) + end + sliceQuant = reverse(sliceQuant, dims=2) + save_bitmap(joinpath("public", "MSI_$(text_nmass).bmp"), sliceQuant, ViridisPalette) + + imgInt = "/MSI_$(text_nmass).bmp?t=$(timestamp)" + plotdataImg, plotlayoutImg, imgWidth, imgHeight = loadImgPlot(imgInt) + current_msi = "MSI_$(text_nmass).bmp" + msgimg = "Image with the Nmass of $(replace(text_nmass, "_" => "."))" + + levels = range(0, maximum(slice), length=8) + Colorbar(fig[1, 1], colormap=cgrad(:viridis, colorLevel, categorical=true), limits=(0, maximum(slice)), ticks=levels, tickformat=log_tick_formatter, label="Intensity", size=25) + save("public/colorbar_MSI_$(text_nmass).png", fig) + colorbar = "/colorbar_MSI_$(text_nmass).png?t=$(timestamp)" + current_col_msi = "colorbar_MSI_$(text_nmass).png" + + msi_bmp = sort(filter(filename -> startswith(filename, "MSI_") && endswith(filename, ".bmp"), readdir("public")), lt=natural) + col_msi_png = sort(filter(filename -> startswith(filename, "colorbar_MSI_") && endswith(filename, ".png"), readdir("public")), lt=natural) + selectedTab = "tab0" + fTime = time() + eTime = round(fTime - sTime, digits=3) + msg = "The image has been created in $(eTime) seconds successfully inside the 'public' folder of the app" end catch e - msg="Warning: $e" + msg = "There was an error creating the image: $e" + warning_msg = true + @error "Image creation failed" exception=(e, catch_backtrace()) end - sliceQuant=reverse(sliceQuant, dims=2) - SaveBitmapCl(joinpath("public", "MSI_$(text_nmass).bmp"),sliceQuant,ViridisPalette) - # Use timestamp to refresh image interface container - imgInt="/MSI_$(text_nmass).bmp?t=$(timestamp)" - plotdataImg, plotlayoutImg, imgWidth, imgHeight=loadImgPlot(imgInt) - # Get current image - current_msi="MSI_$(text_nmass).bmp" - msgimg="Image with the Nmass of $(replace(text_nmass, "_" => "."))" - # Create colorbar - levels=range(0,maximum(slice),length=8) - Colorbar(fig[1, 1], colormap=cgrad(:viridis, colorLevel, categorical=true), limits=(0, maximum(slice)),ticks=levels,tickformat=log_tick_formatter, label="Intensity", size=25) - save("public/colorbar_MSI_$(text_nmass).png", fig) - colorbar="/colorbar_MSI_$(text_nmass).png?t=$(timestamp)" - # Get current colorbar - current_col_msi="colorbar_MSI_$(text_nmass).png" - # We update the directory to include the new placed images. - msi_bmp=sort(filter(filename -> startswith(filename, "MSI_") && endswith(filename, ".bmp"), readdir("public")),lt=natural) - col_msi_png=sort(filter(filename -> startswith(filename, "colorbar_MSI_") && endswith(filename, ".png"), readdir("public")),lt=natural) - selectedTab="tab0" - fTime=time() - eTime=round(fTime-sTime,digits=3) - msg="The file has been created in $(eTime) seconds successfully inside the 'public' folder of the app" + else + msg = "Invalid parameters. Nmass, Tol, or colorLevel are incorrect." + warning_msg = true + @error msg end - catch e - msg="There was an error loading the ImzML file, please verify the file accordingly and try again. $(e)" - warning_msg=true + finally + # This block will always run at the end of the async task + GC.gc() + if Sys.islinux() + ccall(:malloc_trim, Int32, (Int32,), 0) + end + btnStartDisable = false + btnPlotDisable = false + btnOpticalDisable = false + progress = false + btnSpectraDisable = false + SpectraEnabled = true end - else - msg="File does not exist or a parameter is incorrect, please try again." - warning_msg=true - end - GC.gc() # Trigger garbage collection - if Sys.islinux() - ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure julia returns the freed memory to OS - end - btnStartDisable=false - btnPlotDisable=false - btnOpticalDisable=false - progress=false - if isfile(full_routeMz) - # We enable coord search and spectra plot creation - btnSpectraDisable=false - SpectraEnabled=true end end @onbutton createMeanPlot begin - msg="Mean spectrum plot selected" - sTime=time() - if isfile(full_routeMz) # Check if the file exists - progressSpectraPlot=true - btnPlotDisable=true - btnStartDisable=true - msg="Loading plot..." - plotdata, plotlayout, xSpectraMz, ySpectraMz=meanSpectrumPlot(full_routeMz) - GC.gc() # Trigger garbage collection - if Sys.islinux() - ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure julia returns the freed memory to OS + if msi_data === nothing + msg = "No data loaded. Please select a file first." + warning_msg = true + return + end + + # UI updates immediately + progressSpectraPlot = true + btnPlotDisable = true + btnStartDisable = true + msg = "Loading plot..." + + @async begin + try + sTime = time() + plotdata, plotlayout, xSpectraMz, ySpectraMz = meanSpectrumPlot(msi_data) + + selectedTab = "tab2" + fTime = time() + eTime = round(fTime - sTime, digits=3) + msg = "Plot loaded in $(eTime) seconds" + catch e + msg = "Could not generate mean spectrum plot: $e" + warning_msg = true + @error "Mean spectrum plotting failed" exception=(e, catch_backtrace()) + finally + # This runs after the async task is finished + progressSpectraPlot = false + btnPlotDisable = false + btnSpectraDisable = false + if msi_data !== nothing && msi_data.source isa ImzMLSource + btnStartDisable = false + end end - selectedTab="tab2" - fTime=time() - eTime=round(fTime-sTime,digits=3) - msg="Plot loaded in $(eTime) seconds" - else - msg="there was an error with the mzML, please try again" - warning_msg=true end - progressSpectraPlot=false - btnPlotDisable=false - if endswith(full_route, "imzML") - btnStartDisable=false + end + + @onbutton createSumPlot begin + if msi_data === nothing + msg = "No data loaded. Please select a file first." + warning_msg = true + return end - if isfile(full_routeMz) - # We enable coord search and spectra plot creation - btnSpectraDisable=false - SpectraEnabled=true + + # UI updates immediately + progressSpectraPlot = true + btnPlotDisable = true + btnStartDisable = true + msg = "Loading total spectrum plot..." + + @async begin + try + sTime = time() + plotdata, plotlayout, xSpectraMz, ySpectraMz = sumSpectrumPlot(msi_data) + + selectedTab = "tab2" + fTime = time() + eTime = round(fTime - sTime, digits=3) + msg = "Total plot loaded in $(eTime) seconds" + catch e + msg = "Could not generate total spectrum plot: $e" + warning_msg = true + @error "Total spectrum plotting failed" exception=(e, catch_backtrace()) + finally + # This runs after the async task is finished + progressSpectraPlot = false + btnPlotDisable = false + btnSpectraDisable = false + if msi_data !== nothing && msi_data.source isa ImzMLSource + btnStartDisable = false + end + end end end @onbutton createXYPlot begin - msg="XY spectrum plot selected" - sTime=time() - if isfile(full_routeMz) # Check if the file exists - progressSpectraPlot=true - btnStartDisable=true - btnPlotDisable=true - btnSpectraDisable=true - msg="Loading plot..." - spectraMz=LoadMzml(full_routeMz) - layoutSpectra=PlotlyBase.Layout( - title="($xCoord, $yCoord) Specific spectrum plot", - xaxis=PlotlyBase.attr( - title="m/z", - showgrid=true - ), - yaxis=PlotlyBase.attr( - title="Intensity", - showgrid=true - ), - autosize=false, - margin=attr(l=0,r=0,t=120,b=0,pad=0) - ) - if xCoord < 1 - xCoord=1 - elseif xCoord > imgWidth - xCoord=imgWidth - end - if yCoord > -1 - yCoord=-1 - elseif yCoord < -imgHeight - yCoord=-imgHeight - end - xSpectraMz=spectraMz[1,abs(xCoord)] - ySpectraMz=spectraMz[2,abs(yCoord)] - traceSpectra=PlotlyBase.scatter(x=xSpectraMz, y=ySpectraMz, mode="lines") - plotdata=[traceSpectra] # We add the data from spectra to the plot - plotlayout=layoutSpectra - GC.gc() # Trigger garbage collection - if Sys.islinux() - ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure julia returns the freed memory to OS - end - selectedTab="tab2" - fTime=time() - eTime=round(fTime-sTime,digits=3) - msg="Plot loaded in $(eTime) seconds" - else - msg="there was an error with the mzML or the coordenates, please try again" - warning_msg=true + if msi_data === nothing + msg = "No data loaded. Please select a file first." + warning_msg = true + return end - progressSpectraPlot=false - btnPlotDisable=false - if endswith(full_route, "imzML") - btnStartDisable=false - end - if isfile(full_routeMz) - # We enable coord search and spectra plot creation - btnSpectraDisable=false - SpectraEnabled=true + + # UI updates immediately + progressSpectraPlot = true + btnStartDisable = true + btnPlotDisable = true + btnSpectraDisable = true + msg = "Loading plot..." + + @async begin + try + sTime = time() + # The UI uses negative Y values, so we adjust before calling the plot function + y = yCoord < 0 ? abs(yCoord) : yCoord + plotdata, plotlayout, xSpectraMz, ySpectraMz = xySpectrumPlot(msi_data, xCoord, y, imgWidth, imgHeight) + + # Update UI coordinates + xCoord = plotlayout.title == "Spectrum #$(xCoord)" ? xCoord : clamp(xCoord, 1, imgWidth) + yCoord = plotlayout.title == "Spectrum #$(xCoord)" ? 0 : -clamp(y, 1, imgHeight) + + selectedTab = "tab2" + + fTime = time() + eTime = round(fTime - sTime, digits=3) + msg = "Plot loaded in $(eTime) seconds" + catch e + msg = "Could not retrieve spectrum: $e" + warning_msg = true + @error "Spectrum plotting failed" exception=(e, catch_backtrace()) + finally + # This runs after the async task is finished + progressSpectraPlot = false + btnPlotDisable = false + btnSpectraDisable = false + if msi_data !== nothing && msi_data.source isa ImzMLSource + btnStartDisable = false + end + end end end @@ -796,14 +792,21 @@ include("./julia_imzML_visual.jl") cleaned_imgInt=replace(imgInt, r"\?.*" => "") cleaned_imgInt=lstrip(cleaned_imgInt, '/') var=joinpath( "./public", cleaned_imgInt ) - sTime=time() - if isfile(var) - progressPlot=true - btnPlotDisable=true - btnStartDisable=true - btnSpectraDisable=true + if !isfile(var) + msg="Image could not be 3d plotted" + warning_msg=true + return + end + + progressPlot=true + btnPlotDisable=true + btnStartDisable=true + btnSpectraDisable=true + + @async begin try + sTime=time() plotdata3d, plotlayout3d=loadSurfacePlot(imgInt) GC.gc() # Trigger garbage collection if Sys.islinux() @@ -816,34 +819,39 @@ include("./julia_imzML_visual.jl") catch e msg="Failed to load and process image: $e" warning_msg=true + finally + progressPlot=false + btnPlotDisable=false + btnStartDisable=false + if msi_data !== nothing + # We enable coord search and spectra plot creation + btnSpectraDisable=false + SpectraEnabled=true + end end - else - msg="Image could not be 3d plotted" - warning_msg=true end - progressPlot=false - btnPlotDisable=false - btnStartDisable=false - if isfile(full_routeMz) - # We enable coord search and spectra plot creation - btnSpectraDisable=false - SpectraEnabled=true - end - end - # 3d plot for TrIQ + end # 3d plot for TrIQ + @onbutton triq3dPlot begin msg="TrIQ 3D plot selected" cleaned_imgIntT=replace(imgIntT, r"\?.*" => "") cleaned_imgIntT=lstrip(cleaned_imgIntT, '/') var=joinpath( "./public", cleaned_imgIntT ) - sTime=time() - if isfile(var) - progressPlot=true - btnPlotDisable=true - btnStartDisable=true - btnSpectraDisable=true + if !isfile(var) + msg="Image could not be 3d plotted" + warning_msg=true + return + end + + progressPlot=true + btnPlotDisable=true + btnStartDisable=true + btnSpectraDisable=true + + @async begin try + sTime=time() plotdata3d, plotlayout3d=loadSurfacePlot(imgIntT) GC.gc() # Trigger garbage collection if Sys.islinux() @@ -856,18 +864,16 @@ include("./julia_imzML_visual.jl") catch e msg="Failed to load and process image: $e" warning_msg=true + finally + progressPlot=false + btnPlotDisable=false + btnStartDisable=false + if msi_data !== nothing + # We enable coord search and spectra plot creation + btnSpectraDisable=false + SpectraEnabled=true + end end - else - msg="Image could not be 3d plotted" - warning_msg=true - end - progressPlot=false - btnPlotDisable=false - btnStartDisable=false - if isfile(full_routeMz) - # We enable coord search and spectra plot creation - btnSpectraDisable=false - SpectraEnabled=true end end @@ -877,15 +883,21 @@ include("./julia_imzML_visual.jl") cleaned_imgInt=replace(imgInt, r"\?.*" => "") cleaned_imgInt=lstrip(cleaned_imgInt, '/') var=joinpath("./public", cleaned_imgInt) - sTime=time() - - if isfile(var) - progressPlot=true - btnPlotDisable=true - btnStartDisable=true - btnSpectraDisable=true + + if !isfile(var) + msg="Image could not be 2D plotted" + warning_msg=true + return + end + + progressPlot=true + btnPlotDisable=true + btnStartDisable=true + btnSpectraDisable=true + + @async begin try - img=load(var) + sTime=time() plotdataC,plotlayoutC=loadContourPlot(imgInt) GC.gc() # Trigger garbage collection if Sys.islinux() @@ -898,18 +910,16 @@ include("./julia_imzML_visual.jl") catch e msg="Failed to load and process image: $e" warning_msg=true + finally + progressPlot=false + btnPlotDisable=false + btnStartDisable=false + if msi_data !== nothing + # We enable coord search and spectra plot creation + btnSpectraDisable=false + SpectraEnabled=true + end end - else - msg="Image could not be 2D plotted" - warning_msg=true - end - progressPlot=false - btnPlotDisable=false - btnStartDisable=false - if isfile(full_routeMz) - # We enable coord search and spectra plot creation - btnSpectraDisable=false - SpectraEnabled=true end end # Contour 2d plot for TrIQ @@ -918,15 +928,21 @@ include("./julia_imzML_visual.jl") cleaned_imgIntT=replace(imgIntT, r"\?.*" => "") cleaned_imgIntT=lstrip(cleaned_imgIntT, '/') var=joinpath("./public", cleaned_imgIntT) - sTime=time() - - if isfile(var) - progressPlot=true - btnPlotDisable=true - btnStartDisable=true - btnSpectraDisable=true + + if !isfile(var) + msg="Image could not be 2D plotted" + warning_msg=true + return + end + + progressPlot=true + btnPlotDisable=true + btnStartDisable=true + btnSpectraDisable=true + + @async begin try - img=load(var) + sTime=time() plotdataC,plotlayoutC=loadContourPlot(imgIntT) GC.gc() # Trigger garbage collection if Sys.islinux() @@ -939,18 +955,16 @@ include("./julia_imzML_visual.jl") catch e msg="Failed to load and process image: $e" warning_msg=true + finally + progressPlot=false + btnPlotDisable=false + btnStartDisable=false + if msi_data !== nothing + # We enable coord search and spectra plot creation + btnSpectraDisable=false + SpectraEnabled=true + end end - else - msg="Image could not be 2D plotted" - warning_msg=true - end - progressPlot=false - btnPlotDisable=false - btnStartDisable=false - if isfile(full_routeMz) - # We enable coord search and spectra plot creation - btnSpectraDisable=false - SpectraEnabled=true end end @@ -961,48 +975,39 @@ include("./julia_imzML_visual.jl") # To include a visualization in the spectrum plot indicating where is the selected mass @onchange Nmass begin if !isempty(xSpectraMz) - traceSpectra=PlotlyBase.scatter(x=xSpectraMz, y=ySpectraMz, mode="lines",name="Spectra",showlegend=false) - trace2=PlotlyBase.scatter(x=[Nmass, Nmass],y=[0, maximum(ySpectraMz)],mode="lines",line=attr(color="red", width=0.5),name="m/z selected",showlegend=false) - plotdata=[traceSpectra,trace2] # We add the data from spectra and the red line to the plot + # Use a stem plot for the main spectrum for consistency + traceSpectra = PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz, marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="m/z: %{x:.4f}", showlegend=false) + + # Keep this as a scatter plot to draw the vertical line + trace2 = PlotlyBase.scatter(x=[Nmass, Nmass], y=[0, maximum(ySpectraMz)], mode="lines", line=attr(color="red", width=0.5), name="m/z selected", showlegend=false) + + plotdata = [traceSpectra, trace2] end end # Event detection for clicking on the images @onchange data_click begin + if selectedTab == "tab1" || selectedTab == "tab0" + # This is for the image heatmaps + cursor_data = data_click["cursor"] + x = Int32(round(cursor_data["x"])) + y = Int32(round(cursor_data["y"])) # y is negative in the UI + + # Update the reactive coordinates, which will trigger the crosshair update + xCoord = clamp(x, 1, imgWidth) + yCoord = clamp(y, -imgHeight, -1) + end + end + + @onchange xCoord, yCoord begin if selectedTab == "tab1" - cursor_data=data_click["cursor"] - xCoord=Int32(round(cursor_data["x"])) - yCoord=Int32(round(cursor_data["y"])) - if xCoord < 1 - xCoord=1 - elseif xCoord > imgWidth - xCoord=imgWidth - end - if yCoord > -1 - yCoord=-1 - elseif yCoord < -imgHeight - yCoord=-imgHeight - end # Get the x and y values from the click of the cursor and make sure they don't exceed image proportions - plotdataImgT=filter(trace -> !(get(trace, :name, "") in ["Line X", "Line Y"]), plotdataImgT) - trace1, trace2=crossLinesPlot(xCoord, yCoord, imgWidth, -imgHeight) - plotdataImgT=append!(plotdataImgT, [trace1, trace2]) + plotdataImgT = filter(trace -> !(get(trace, :name, "") in ["Line X", "Line Y"]), plotdataImgT) + trace1, trace2 = crossLinesPlot(xCoord, yCoord, imgWidth, -imgHeight) + plotdataImgT = append!(plotdataImgT, [trace1, trace2]) elseif selectedTab == "tab0" - cursor_data=data_click["cursor"] - xCoord=Int32(round(cursor_data["x"])) - yCoord=Int32(round(cursor_data["y"])) - if xCoord < 1 - xCoord=1 - elseif xCoord > imgWidth - xCoord=imgWidth - end - if yCoord > -1 - yCoord=-1 - elseif yCoord < -imgHeight - yCoord=-imgHeight - end # Get the x and y values from the click of the cursor and make sure they don't exceed image proportions - plotdataImg=filter(trace -> !(get(trace, :name, "") in ["Line X", "Line Y","Optical"]), plotdataImg) - trace1, trace2=crossLinesPlot(xCoord, yCoord, imgWidth, -imgHeight) - plotdataImg=append!(plotdataImg, [trace1, trace2]) + plotdataImg = filter(trace -> !(get(trace, :name, "") in ["Line X", "Line Y", "Optical"]), plotdataImg) + trace1, trace2 = crossLinesPlot(xCoord, yCoord, imgWidth, -imgHeight) + plotdataImg = append!(plotdataImg, [trace1, trace2]) end end diff --git a/app.jl.html b/app.jl.html index 77c2180..f9c9653 100644 --- a/app.jl.html +++ b/app.jl.html @@ -69,6 +69,11 @@ Mean spectrum plot + + + Sum Spectrum plot + + Spectrum plot (X,Y) diff --git a/julia_imzML_visual.jl b/julia_imzML_visual.jl index 604fcb1..e20abf7 100644 --- a/julia_imzML_visual.jl +++ b/julia_imzML_visual.jl @@ -1,84 +1,3 @@ -# IntQuantCl is originally a function of julia mzMl imzML with the adition -# of altering the scale of the colors according to colorlevel. -function IntQuantCl( slice , colorLevel) - # Compute scale factor for amplitude discretization - lower = minimum( slice ) - scale = colorLevel / maximum( slice ) - dim = size( slice ) - image = zeros( UInt8, dim[1], dim[2] ) - for i in 1:length( slice ) - image[i] = convert( UInt8, floor( slice[i] * scale + 0.5 ) ) - end - return image -end - -# SaveBitmap originally a function of the mzML imzML library in julia, -# This function dinamically adjust the color palete adjusting to the ammount of colors -# available in pixmap -function SaveBitmapCl( name, pixMap::Array{UInt8,2}, colorTable::Array{UInt32,1} ) - # Get image dimensions - dim = size( pixMap ) - if length( dim ) != 2 - return 0 - end - # Normalize pixel values to get a more accurate reading of the image - minVal = minimum(pixMap) - maxVal = maximum(pixMap) - pixMap = round.(UInt8, 255 * (pixMap .- minVal) ./ (maxVal - minVal)) - # Compute row padding - padding = ( 4 - dim[1] & 0x3 ) & 0x3 - # Compute file dimensions. Header = 14 + 40 + ( 256 * 4 ) = 1078 - offset = 1078 - imgBytes = dim[2] * ( dim[1] + padding ) - # Create file - stream = open( name, "w" ) - # Save file header - write( stream, UInt16( 0x4D42 ) ) - write( stream, UInt32[ offset + imgBytes, 0 , offset ] ) - # Save info header - write( stream, UInt32[ 40, dim[1], dim[2], 0x80001, 0 ] ) - write( stream, UInt32[ imgBytes, 0, 0, 256, 0 ] ) - # Save color table - write( stream, colorTable ) - if length( colorTable ) < 256 - fixTable = zeros( UInt32, 256 - length( colorTable ) ) - write( stream, fixTable ) - end - # Save image pixels - if padding == 0 - for i = 1:dim[2] - write( stream, pixMap[:,i] ) - end - else - zeroPad = zeros( UInt8, padding ) - for i in 1:dim[2] - write( stream, pixMap[:,i] ) - write( stream, zeroPad ) - end - end - # Close file - close( stream ) -end - -# SaveBitmap originally a function of the mzML imzML library in julia, -# now has an adjustment for NaN values in case they exist to mantain data integrity -function GetMzSliceJl(imzML, mass, tolerance) - # Alloc space for slice - width = maximum(imzML[1, :]) - height = maximum(imzML[2, :]) - image = fill(0.0, width, height) - - for i in 1:size(imzML)[2] - index = julia_mzML_imzML.FindMass(imzML[3, i], mass, tolerance) - if index != 0 - image[imzML[1, i], imzML[2, i]] = imzML[4, i][index] - end - end - # Adjustment for NaN values with 0 - replace!(image, NaN => 0.0) - return image -end - # == Search functions == # Functions that recieve a list to update, and the current direction both as string for # searching in the directory the position the list is going @@ -129,10 +48,12 @@ function loadImgPlot(interfaceImg::String) layout=PlotlyBase.Layout( xaxis=PlotlyBase.attr( visible=false, - scaleanchor="y" + scaleanchor="y", + range=[0, width] ), yaxis=PlotlyBase.attr( - visible=false + visible=false, + range=[-height, 0] ), margin=attr(l=0,r=0,t=0,b=0,pad=0) ) @@ -167,6 +88,7 @@ function loadImgPlot(interfaceImg::String) plotlayout=layout return plotdata, plotlayout, width, height end + # loadImgPlot recieves the local directory of the image as a string, the local directory o the overlay image # and the transparency its required to have. Returns the layout and data for the heatmap plotly plot # this function loads the image into a plot @@ -286,6 +208,7 @@ function loadContourPlot(interfaceImg::String) plotlayout=layout return plotdata, plotlayout end + # loadSurfacePlot recieves the local directory of the image as a string, # returns the layout and data for the surface plotly plot # this function loads the image and applies a gaussian filter @@ -363,6 +286,7 @@ function loadSurfacePlot(interfaceImg::String) plotlayout=layout3D return plotdata, plotlayout end + # This function recieves the x and y coords currently selected, and the dimentions of # the image to create two traces that will display in a cross section function crossLinesPlot(x, y, maxwidth, maxheight) @@ -404,77 +328,117 @@ function log_tick_formatter(values::Vector{Float64}) end -# Median filter: an adaptation of the R medianfilter, which averages the matrix -# with the close pixels just from the sides to reduce noise. -# This one in particular is a midpoint fiter from a 3x3 neighbour area -function medianFilterjl(pixMap) - height, width = size(pixMap) - padded_pixMap = padarray(pixMap, 1) - target = zeros(eltype(pixMap), height, width) - - for j in 1:width - for i in 1:height - neighbors = [] - for dj in j:j+2 - for di in i:i+2 - push!(neighbors, padded_pixMap[di, dj]) - end - end - target[i, j] = median(neighbors) - end - end - - return target -end - -function padarray(A, padsize) - h, w = size(A) - padded = zeros(eltype(A), h + 2*padsize, w + 2*padsize) - padded[padsize+1:end-padsize, padsize+1:end-padsize] .= A - padded[1:padsize, padsize+1:end-padsize] .= A[1:padsize, :] - padded[end-padsize+1:end, padsize+1:end-padsize] .= A[end-padsize+1:end, :] - padded[padsize+1:end-padsize, 1:padsize] .= A[:, 1:padsize] - padded[padsize+1:end-padsize, end-padsize+1:end] .= A[:, end-padsize+1:end] - padded[1:padsize, 1:padsize] .= A[1, 1] - padded[1:padsize, end-padsize+1:end] .= A[1, end] - padded[end-padsize+1:end, 1:padsize] .= A[end, 1] - padded[end-padsize+1:end, end-padsize+1:end] .= A[end, end] - return padded -end - # meanSpectrumPlot recieves the local directory of the image as a string, # returns the layout and data for the surface plotly plot # this function loads the spectra data and makes a mean to display # its values in the spectrum plot -function meanSpectrumPlot(mzmlRoute::String) - spectraMz=LoadMzml(mzmlRoute) - xSpectraMz=Float64[] - ySpectraMz=Float64[] - - layout=PlotlyBase.Layout( - title="Mean spectrum plot", +function meanSpectrumPlot(data::MSIData) + layout = PlotlyBase.Layout( + title="Average Spectrum Plot", + hovermode="closest", xaxis=PlotlyBase.attr( title="m/z", + showgrid=true + ), + yaxis=PlotlyBase.attr( + title="Average Intensity", showgrid=true, - tickformat = ".3g" + tickformat=".3g" + ), + margin=attr(l=0, r=0, t=120, b=0, pad=0) + ) + + # Use the new, efficient function from the backend + xSpectraMz, ySpectraMz = get_average_spectrum(data) + + if isempty(xSpectraMz) + @warn "Average spectrum is empty." + trace = PlotlyBase.stem(x=Float64[], y=Float64[]) + else + trace = PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz, marker=attr(size=1, color="blue", opacity=0.5), name="Average", hoverinfo="x",hovertemplate="m/z: %{x:.4f}") + end + + + plotdata = [trace] + plotlayout = layout + return plotdata, plotlayout, xSpectraMz, ySpectraMz +end + +function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int, imgHeight::Int) + local mz::AbstractVector, intensity::AbstractVector + local plot_title::String + + is_imaging = data.source isa ImzMLSource + + if is_imaging + # For imaging data, use (X, Y) coordinates + x = clamp(xCoord, 1, imgWidth) + y = clamp(yCoord, 1, imgHeight) + + mz, intensity = GetSpectrum(data, x, y) + plot_title = "Spectrum at ($x, $y)" + else + # For non-imaging data, treat xCoord as the spectrum index + index = clamp(xCoord, 1, length(data.spectra_metadata)) + + mz, intensity = GetSpectrum(data, index) + plot_title = "Spectrum #$index" + end + + layout = PlotlyBase.Layout( + title=plot_title, + hovermode="closest", + xaxis=PlotlyBase.attr( + title="m/z", + showgrid=true ), yaxis=PlotlyBase.attr( title="Intensity", showgrid=true, - tickformat = ".3g" + tickformat=".3g" ), - autosize=false, - margin=attr(l=0,r=0,t=120,b=0,pad=0) + margin=attr(l=0, r=0, t=120, b=0, pad=0) ) - try - xSpectraMz=mean(spectraMz[1,:]) - ySpectraMz=mean(spectraMz[2,:]) - catch e - xSpectraMz=spectraMz[1,1] - ySpectraMz=spectraMz[2,1] + + # Downsample for plotting performance + mz_down, int_down = MSI_src.downsample_spectrum(mz, intensity) + + trace = PlotlyBase.stem(x=mz_down, y=int_down, marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="m/z: %{x:.4f}") + + plotdata = [trace] + plotlayout = layout + + # Return the full data for other uses, and the plot data + return plotdata, plotlayout, mz, intensity +end + +function sumSpectrumPlot(data::MSIData) + layout = PlotlyBase.Layout( + title="Total Spectrum Plot", + hovermode="closest", + xaxis=PlotlyBase.attr( + title="m/z", + showgrid=true + ), + yaxis=PlotlyBase.attr( + title="Total Intensity", + showgrid=true, + tickformat=".3g" + ), + margin=attr(l=0, r=0, t=120, b=0, pad=0) + ) + + # Use the get_total_spectrum function from the backend + xSpectraMz, ySpectraMz = get_total_spectrum(data) + + if isempty(xSpectraMz) + @warn "Total spectrum is empty." + trace = PlotlyBase.stem(x=Float64[], y=Float64[]) + else + trace = PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz, marker=attr(size=1, color="blue", opacity=0.5), name="Total", hoverinfo="x",hovertemplate="m/z: %{x:.4f}") end - trace=PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz,marker=attr(size=1, color="blue", opacity=0.1)) - plotdata=[trace] # We add the data from spectra to the plot - plotlayout=layout + + plotdata = [trace] + plotlayout = layout return plotdata, plotlayout, xSpectraMz, ySpectraMz -end \ No newline at end of file +end diff --git a/src/MSIData.jl b/src/MSIData.jl index 4ac7107..b570ee3 100644 --- a/src/MSIData.jl +++ b/src/MSIData.jl @@ -6,7 +6,7 @@ including caching and iteration logic, for handling large mzML and imzML dataset efficiently. """ -using Base64, Libz # For reading binary data +using Base64, Libz, Serialization # For reading binary data # Abstract type for different data sources (e.g., mzML, imzML) # This allows dispatching to the correct binary reading logic. @@ -179,17 +179,10 @@ function GetSpectrum(data::MSIData, x::Int, y::Int) return GetSpectrum(data, index) # Call the existing method end -""" - get_total_spectrum(msi_data::MSIData; num_bins::Int=20000) -> Tuple{Vector{Float64}, Vector{Float64}} +using Serialization -Calculates the sum of all spectra in the dataset by binning. -The m/z range is determined dynamically by finding the global min/max m/z values -across the entire dataset. - -Returns a tuple containing two vectors: the binned m/z axis and the summed intensities. -""" -function get_total_spectrum(msi_data::MSIData; num_bins::Int=20000) - println("Calculating total spectrum...") +function get_total_spectrum_imzml(msi_data::MSIData; num_bins::Int=2000) + println("Calculating total spectrum (2-pass method for imzML)...") # 1. First Pass: Find the global m/z range println(" Pass 1: Finding global m/z range...") @@ -222,7 +215,6 @@ function get_total_spectrum(msi_data::MSIData; num_bins::Int=20000) return # continue equivalent end for i in eachindex(mz) - # Find the correct bin for the current m/z value bin_index = clamp(round(Int, (mz[i] - global_min_mz) / bin_step) + 1, 1, num_bins) intensity_sum[bin_index] += intensity[i] end @@ -232,6 +224,83 @@ function get_total_spectrum(msi_data::MSIData; num_bins::Int=20000) return (collect(mz_bins), intensity_sum) end +function get_total_spectrum_mzml(msi_data::MSIData; num_bins::Int=2000) + println("Calculating total spectrum (single-pass optimization for mzML)...") + num_spectra = length(msi_data.spectra_metadata) + if num_spectra == 0 + return (Float64[], Float64[]) + end + + temp_path, temp_io = mktemp() + try + # --- Pass 1: Write spectra to temp file and find min/max m/z --- + println(" Pass 1: Caching spectra and finding global m/z range...") + global_min_mz = Inf + global_max_mz = -Inf + + _iterate_spectra_fast(msi_data) do idx, mz, intensity + Serialization.serialize(temp_io, (mz, intensity)) + if !isempty(mz) + local_min, local_max = extrema(mz) + global_min_mz = min(global_min_mz, local_min) + global_max_mz = max(global_max_mz, local_max) + end + end + + flush(temp_io) + + if !isfinite(global_min_mz) + @warn "Could not determine a valid m/z range. All spectra might be empty." + return (Float64[], Float64[]) + end + println(" Global m/z range found: [$(global_min_mz), $(global_max_mz)]") + + # --- Pass 2: Read from temp file and bin intensities --- + println(" Pass 2: Reading from cache and summing intensities into $num_bins bins...") + + seekstart(temp_io) + mz_bins = range(global_min_mz, stop=global_max_mz, length=num_bins) + intensity_sum = zeros(Float64, num_bins) + bin_step = step(mz_bins) + + while !eof(temp_io) + mz, intensity = Serialization.deserialize(temp_io)::Tuple{AbstractVector, AbstractVector} + + if isempty(mz) + continue + end + for i in eachindex(mz) + bin_index = clamp(round(Int, (mz[i] - global_min_mz) / bin_step) + 1, 1, num_bins) + intensity_sum[bin_index] += intensity[i] + end + end + + println("Total spectrum calculation complete.") + return (collect(mz_bins), intensity_sum) + + finally + close(temp_io) + rm(temp_path, force=true) + end +end + +""" + get_total_spectrum(msi_data::MSIData; num_bins::Int=20000) -> Tuple{Vector{Float64}, Vector{Float64}} + +Calculates the sum of all spectra in the dataset by binning. +This function dispatches to a specialized implementation based on the file type +(.imzML or .mzML) for optimal performance. + +Returns a tuple containing two vectors: the binned m/z axis and the summed intensities. +""" +function get_total_spectrum(msi_data::MSIData; num_bins::Int=2000) + if msi_data.source isa ImzMLSource + return get_total_spectrum_imzml(msi_data, num_bins=num_bins) + else # MzMLSource + return get_total_spectrum_mzml(msi_data, num_bins=num_bins) + end +end + """ get_average_spectrum(msi_data::MSIData; num_bins::Int=20000) -> Tuple{Vector{Float64}, Vector{Float64}} @@ -241,7 +310,7 @@ This is effectively the total ion chromatogram (TIC) divided by the number of sp Returns a tuple containing two vectors: the binned m/z axis and the averaged intensities. """ -function get_average_spectrum(msi_data::MSIData; num_bins::Int=20000) +function get_average_spectrum(msi_data::MSIData; num_bins::Int=2000) # This function uses the exact same logic as get_total_spectrum... mz_bins, intensity_sum = get_total_spectrum(msi_data, num_bins=num_bins) @@ -365,5 +434,3 @@ function _iterate_spectra_fast(f::Function, data::MSIData) # Dispatch to the correct implementation based on the source type _iterate_spectra_fast_impl(f, data, data.source) end - - diff --git a/src/MzmlConverter.jl b/src/MzmlConverter.jl index 69e10be..695eed4 100644 --- a/src/MzmlConverter.jl +++ b/src/MzmlConverter.jl @@ -26,26 +26,19 @@ struct BinaryMetadata end """ - GetMzmlScanTime(fileName::String) + GetMzmlScanTime_linebyline(fileName::String) -Parses a .mzML file to extract the scan start time for each spectrum. - -# Arguments -* `fileName`: Path to the .mzML file. - -# Returns -- A `Matrix{Int64}` where each row is `[spectrum_index, time_in_milliseconds]`. +(Internal) Slow, line-by-line parser for scan times. Used as a fallback if +the .mzML file index is missing or corrupt. """ -function GetMzmlScanTime(fileName::String) +function GetMzmlScanTime_linebyline(fileName::String) times = Tuple{Int64, Int64}[] - # Pre-allocate based on an estimate to reduce re-allocations try file_size = filesize(fileName) - estimated_spectra = max(1000, file_size ÷ 10000) # Heuristic + estimated_spectra = max(1000, file_size ÷ 10000) sizehint!(times, estimated_spectra) catch - # Ignore if filesize fails, sizehint! is just an optimization end open(fileName, "r") do stream @@ -60,27 +53,23 @@ function GetMzmlScanTime(fileName::String) current_index = 0 current_time = nothing - # Process index from the start tag itself idx_match = match(r"index=\"(\d+)\"", line) if idx_match !== nothing current_index = parse(Int, idx_match.captures[1]) + 1 end end elseif state == :in_spectrum - # Look for time cvParam if occursin("", line) state = :outside_spectrum if current_index > 0 && current_time !== nothing @@ -91,9 +80,100 @@ function GetMzmlScanTime(fileName::String) end end - # Sort by spectrum index to ensure correct order before normalization sort!(times, by = x -> x[1]) + if !isempty(times) + first_time = times[1][2] + for i in eachindex(times) + times[i] = (times[i][1], times[i][2] - first_time) + end + end + + if isempty(times) + return Matrix{Int64}(undef, 0, 2) + end + return permutedims(hcat(collect.(times)...)) +end + +""" + GetMzmlScanTime(fileName::String) + +Parses a .mzML file to extract the scan start time for each spectrum. +Uses the indexed part of the .mzML file for fast access, falling back to +a slower line-by-line parse if the index is not present. + +# Arguments +* `fileName`: Path to the .mzML file. + +# Returns +- A `Matrix{Int64}` where each row is `[spectrum_index, time_in_milliseconds]`. +""" +function GetMzmlScanTime(fileName::String) + times = Tuple{Int64, Int64}[] + + try + open(fileName, "r") do stream + # 1. Find and parse the spectrum index offsets + seekend(stream) + end_chunk_size = min(filesize(stream), 8192) + seek(stream, filesize(stream) - end_chunk_size) + footer = read(stream, String) + + index_offset_match = match(r"(\d+)", footer) + if index_offset_match === nothing + @warn "No found. Falling back to slow line-by-line parsing for scan times. This may be memory intensive." + return GetMzmlScanTime_linebyline(fileName) + end + + index_offset = parse(Int64, index_offset_match.captures[1]) + seek(stream, index_offset) + + # The find_tag function is defined in ParserHelpers.jl + if find_tag(stream, r"", line) || occursin("", line) + break + end + end + + if current_time !== nothing + push!(times, (idx, current_time)) + end + end + end + catch e + @error "Failed to parse scan times with indexed method. Falling back to line-by-line." exception=(e, catch_backtrace()) + return GetMzmlScanTime_linebyline(fileName) + end + + # The index is already sorted by spectrum index, so no need to sort `times`. + # Normalize times relative to the first scan if !isempty(times) first_time = times[1][2] @@ -102,7 +182,6 @@ function GetMzmlScanTime(fileName::String) end end - # Convert vector of tuples to a matrix if isempty(times) return Matrix{Int64}(undef, 0, 2) end @@ -329,131 +408,113 @@ function RenderPixel(pixel_info, scans, msi_data::MSIData, scan_time_deltas, pix return (mz_array, new_intensity) end -function ConvertMzmlToImzml(source_file::String, timing_matrix::Matrix{Int64}, scans::Matrix{Int64}) +function ConvertMzmlToImzml(source_file::String, target_ibd_file::String, timing_matrix::Matrix{Int64}, scans::Matrix{Int64}) if size(timing_matrix, 1) == 0 - return ProcessedPixel[], (0, 0) + # Create an empty .ibd file if there's nothing to process + open(target_ibd_file, "w") do ibd_stream + write(ibd_stream, zeros(UInt8, 16)) # UUID placeholder + end + return BinaryMetadata[], Tuple{Int, Int}[], (0, 0) end - # Get image dimensions from timing_matrix (assuming columns 1 and 2 are X and Y) width = maximum(timing_matrix[:, 1]) height = maximum(timing_matrix[:, 2]) - # Open the mzML file using the new memory-efficient API msi_data = OpenMSIData(source_file) - # Pre-calculate time deltas robustly scan_time_deltas = zeros(Int64, size(scans, 1)) if size(scans, 1) > 1 for i in 1:(size(scans, 1) - 1) delta = scans[i+1, 2] - scans[i, 2] - scan_time_deltas[i] = max(1, delta) # Ensure delta is at least 1 + scan_time_deltas[i] = max(1, delta) end - scan_time_deltas[end] = max(1, scan_time_deltas[end-1]) # Use previous delta for last scan, ensure at least 1 + scan_time_deltas[end] = max(1, scan_time_deltas[end-1]) end pixel_time_deltas = zeros(Int64, size(timing_matrix, 1)) if size(timing_matrix, 1) > 1 for i in 1:(size(timing_matrix, 1) - 1) - # Assumes time is in the 3rd column of the timing_matrix delta = timing_matrix[i+1, 3] - timing_matrix[i, 3] - pixel_time_deltas[i] = max(1, delta) # Ensure delta is at least 1 + pixel_time_deltas[i] = max(1, delta) end - pixel_time_deltas[end] = max(1, pixel_time_deltas[end-1]) # Use previous delta for last pixel, ensure at least 1 + pixel_time_deltas[end] = max(1, pixel_time_deltas[end-1]) end - processed_pixels = ProcessedPixel[] - sizehint!(processed_pixels, size(timing_matrix, 1)) + binary_meta_vec = BinaryMetadata[] + sizehint!(binary_meta_vec, size(timing_matrix, 1)) + coords_vec = Tuple{Int, Int}[] + sizehint!(coords_vec, size(timing_matrix, 1)) empty_pixel_count = 0 - # DEBUG: Scan coverage analysis - println("DEBUG: Scan coverage analysis:") - covered_pixels_count = 0 - for i in 1:size(timing_matrix, 1) - first_scan = timing_matrix[i, 4] - last_scan = timing_matrix[i, 5] - if first_scan <= last_scan && first_scan >= 1 && last_scan <= size(scans, 1) - covered_pixels_count += 1 + open(target_ibd_file, "w") do ibd_stream + write(ibd_stream, zeros(UInt8, 16)) # UUID placeholder + + for i in 1:size(timing_matrix, 1) + pixel_info = timing_matrix[i, :] + x, y = pixel_info[1], pixel_info[2] + push!(coords_vec, (x, y)) + + first_scan = pixel_info[4] + last_scan = pixel_info[5] + + if first_scan > last_scan || first_scan < 1 || last_scan > size(scans, 1) + empty_pixel_count += 1 + # For empty pixels, offsets point to the current end of file, with zero length + current_pos = position(ibd_stream) + push!(binary_meta_vec, BinaryMetadata(current_pos, 0, current_pos, 0)) + continue + end + + mz, intensity = RenderPixel(pixel_info, scans, msi_data, scan_time_deltas, pixel_time_deltas) + + # Write m/z array + mz_offset = position(ibd_stream) + for val in mz + write(ibd_stream, htol(Float64(val))) + end + mz_length = position(ibd_stream) - mz_offset + + # Write intensity array + int_offset = position(ibd_stream) + for val in intensity + write(ibd_stream, htol(Float32(val))) + end + int_length = position(ibd_stream) - int_offset + + push!(binary_meta_vec, BinaryMetadata(mz_offset, mz_length, int_offset, int_length)) end end - println(" Pixels with potential scans: $covered_pixels_count/$(size(timing_matrix, 1))") - println(" Total scans available: $(size(scans, 1))") - for i in 1:size(timing_matrix, 1) - pixel_info = timing_matrix[i, :] - x = pixel_info[1] - y = pixel_info[2] - first_scan = pixel_info[4] - last_scan = pixel_info[5] - - if first_scan > last_scan || first_scan < 1 || last_scan > size(scans, 1) - empty_pixel_count += 1 - push!(processed_pixels, ProcessedPixel((x, y), Float64[], Float32[])) - continue - end - - mz, intensity = RenderPixel(pixel_info, scans, msi_data, scan_time_deltas, pixel_time_deltas) - push!(processed_pixels, ProcessedPixel((x, y), mz, intensity)) - end - - @info "Found and created $empty_pixel_count empty pixels out of $(size(timing_matrix, 1)) total." - return processed_pixels, (width, height) + @info "Found and processed $empty_pixel_count empty pixels out of $(size(timing_matrix, 1)) total." + return binary_meta_vec, coords_vec, (width, height) end -function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims::Tuple{Int, Int}) +function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, coords::Vector{Tuple{Int, Int}}, dims::Tuple{Int, Int}) ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") - # Ensure we have valid pixels - if isempty(pixels) - @error "No pixels to export" - return false + if isempty(binary_meta) + @warn "No binary metadata to export; creating empty imzML file." + # Still create a valid, empty imzML file end - binary_meta = BinaryMetadata[] - sizehint!(binary_meta, length(pixels)) - try - # Step 1: Write .ibd file with proper binary format - open(ibd_file, "w") do ibd_stream - # Write UUID as first 16 bytes (placeholder) - write(ibd_stream, zeros(UInt8, 16)) + # The .ibd file is now written by ConvertMzmlToImzml. + # This function is only responsible for the .imzML XML metadata file. - # Write in little-endian format (standard for imzML) - for pixel in pixels - # Write m/z array as 64-bit little-endian floats - mz_offset = position(ibd_stream) - for val in pixel.mz - write(ibd_stream, htol(Float64(val))) # Host to little-endian - end - mz_length = position(ibd_stream) - mz_offset - - # Write intensity array as 32-bit little-endian floats - int_offset = position(ibd_stream) - for val in pixel.intensity - write(ibd_stream, htol(Float32(val))) # Host to little-endian - end - int_length = position(ibd_stream) - int_offset - - push!(binary_meta, BinaryMetadata(mz_offset, mz_length, int_offset, int_length)) - end - end - - # Step 2: Write proper .imzML XML file open(target_file, "w") do imzml_stream - # Use indexedmzML wrapper + # XML Header write(imzml_stream, """ """) - # CV List + # CV List, File Description, etc. (static parts) write(imzml_stream, """ """) - - # File Description write(imzml_stream, """ @@ -462,8 +523,6 @@ function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims:: """) - - # Referenceable Param Groups write(imzml_stream, """ @@ -479,24 +538,18 @@ function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims:: """) - - # Sample List write(imzml_stream, """ """) - - # Software List - OPEN SOURCE write(imzml_stream, """ """) - - # Scan Settings IMAGING write(imzml_stream, """ @@ -506,8 +559,6 @@ function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims:: """) - - # Instrument Configuration MALDI/TOF write(imzml_stream, """ @@ -525,8 +576,6 @@ function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims:: """) - - # Data Processing write(imzml_stream, """ @@ -539,18 +588,21 @@ function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims:: # Run and Spectrum List spectrum_offsets = UInt64[] write(imzml_stream, """ - + """) - # Write each spectrum - for (i, pixel) in enumerate(pixels) - meta = binary_meta[i] - x, y = pixel.coords + # Write each spectrum's metadata + for (i, meta) in enumerate(binary_meta) + x, y = coords[i] spectrum_start = position(imzml_stream) push!(spectrum_offsets, spectrum_start) - write(imzml_stream, """ + # Calculate number of points from byte length + mz_points = meta.mz_length ÷ sizeof(Float64) + int_points = meta.int_length ÷ sizeof(Float32) + + write(imzml_stream, """ @@ -563,14 +615,14 @@ function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims:: - + - + @@ -603,10 +655,10 @@ function ExportImzml(target_file::String, pixels::Vector{ProcessedPixel}, dims:: return true catch e - @error "Failed to export imzML files" exception=(e, catch_backtrace()) - # Clean up partial files - isfile(ibd_file) && rm(ibd_file, force=true) + @error "Failed to export imzML metadata file" exception=(e, catch_backtrace()) + # Clean up partial .imzML file isfile(target_file) && rm(target_file, force=true) + # Do not delete the .ibd file as it might be useful for debugging return false end end @@ -630,17 +682,15 @@ function ImportMzmlFile(source_file::String, sync_file::String, target_file::Str println("Step 2: Matching acquisition times...") timing_matrix = MatchAcquireTime(sync_file, scans; img_width=img_width, img_height=img_height) - println("Step 3: Converting spectra...") - processed_pixels, (width, height) = ConvertMzmlToImzml(source_file, timing_matrix, scans) + println("Step 3: Converting spectra and writing .ibd file...") + ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") + binary_meta, coords, (width, height) = ConvertMzmlToImzml(source_file, ibd_file, timing_matrix, scans) # Flip image vertically to match R script output - for i in eachindex(processed_pixels) - x, y = processed_pixels[i].coords - processed_pixels[i] = ProcessedPixel((x, height - y + 1), processed_pixels[i].mz, processed_pixels[i].intensity) - end + flipped_coords = [(x, height - y + 1) for (x, y) in coords] - println("Step 4: Exporting to .imzML/.ibd format...") - success = ExportImzml(target_file, processed_pixels, (width, height)) + println("Step 4: Exporting .imzML metadata file...") + success = ExportImzml(target_file, binary_meta, flipped_coords, (width, height)) if success println("Conversion successful: $target_file") diff --git a/src/imzML.jl b/src/imzML.jl index 8b7ec9f..0426109 100644 --- a/src/imzML.jl +++ b/src/imzML.jl @@ -1,4 +1,4 @@ -using Images, Statistics, CairoMakie, DataFrames, Printf, ColorSchemes +using Images, Statistics, CairoMakie, DataFrames, Printf, ColorSchemes, StatsBase # --- Extracted from imzML.jl --- @@ -347,6 +347,38 @@ function load_slices(folder, masses, tolerance) end +""" + get_mz_slice(data::MSIData, mass::Real, tolerance::Real) + +Extracts an image slice for a given m/z value without plotting. +This is a performant function that iterates through spectra once. + +# Returns +- A `Matrix{Float64}` representing the intensity slice. +""" +function get_mz_slice(data::MSIData, mass::Real, tolerance::Real) + width, height = data.image_dims + slice_matrix = zeros(Float64, height, width) # Note: height, width for (y,x) indexing + + _iterate_spectra_fast(data) do spec_idx, mz_array, intensity_array + meta = data.spectra_metadata[spec_idx] + + intensity = find_mass(mz_array, intensity_array, mass, tolerance) + + if intensity > 0.0 + # Populate the matrix using (y, x) indexing + if 1 <= meta.x <= width && 1 <= meta.y <= height + slice_matrix[meta.y, meta.x] = intensity + end + end + end + + # The original function had a NaN replacement, which is good practice to keep. + replace!(slice_matrix, NaN => 0.0) + return slice_matrix +end + + """ plot_slice(msi_data::MSIData, mass::Float64, tolerance::Float64, output_dir::String; stage_name="slice", bins=256) @@ -486,6 +518,30 @@ function set_pixel_depth(img, bounds, depth) return result end +""" + TrIQ(pixMap, depth, prob=0.98) + +Applies TrIQ (Treshold Intensity Quantization) normalization to an image. +This function first computes dynamic intensity range bounds by identifying outliers +based on a cumulative probability, then sets the pixel depth (quantizes intensities) +within these bounds. + +# Arguments +- `pixMap`: The input image matrix (e.g., a slice from `GetMzSliceJl`). +- `depth`: The number of grey levels (bins) to quantize the intensities into. +- `prob`: The target cumulative probability (e.g., 0.98) used to determine outlier thresholds. + +# Returns +- A new image matrix with intensities quantized to the specified depth within the TrIQ bounds. +""" +function TrIQ(pixMap, depth, prob=0.98) + # Compute new dynamic range + bounds = get_outlier_thres(pixMap, prob) + + # Set intensity dynamic range + return set_pixel_depth(pixMap, bounds, depth) +end + """ norm_slices_hist(slices, bins; prob=0.98) @@ -515,11 +571,73 @@ function norm_slices_hist(slices, bins; prob=0.98) return (norm_img=norm_img, bounds=mass_bounds) # bounds is now a vector, one per mass end +""" + quantize_intensity(slice::AbstractMatrix{<:Real}, levels::Integer=256) + +Linearly scales the intensity values in a slice to a specified number of levels. +The output is an array of `UInt8` values. +This is a modernized version of `IntQuantCl`. +""" +function quantize_intensity(slice::AbstractMatrix{<:Real}, levels::Integer=256) + max_val = maximum(slice) + if max_val <= 0 + return zeros(UInt8, size(slice)) + end + + # Scale relative to the absolute maximum value to preserve the zero point. + # The original logic used 'colorLevel' which was the max value (e.g., 255). + scale = (levels - 1) / max_val + + # round is equivalent to floor(x+0.5) for positive numbers. + # clamp is used for robustness against floating point inaccuracies. + image = round.(UInt8, clamp.(slice .* scale, 0, levels - 1)) + + return image +end + function median_filter(img) # 3x3 median filter implementation return mapwindow(median, img, (3, 3)) end + + +""" + downsample_spectrum(mz, intensity, n_points=2000) + +Reduces the number of points in a spectrum for faster plotting, while preserving peaks. +It divides the m/z range into `n_points` bins and keeps only the most intense point from each bin. +""" +function downsample_spectrum(mz::AbstractVector, intensity::AbstractVector, n_points::Integer=2000) + if isempty(mz) || length(mz) <= n_points + return mz, intensity + end + + mz_min, mz_max = extrema(mz) + bin_width = (mz_max - mz_min) / n_points + + # We use a vector of tuples to store the max intensity and its corresponding mz for each bin + # (max_intensity, mz_value) + bins = fill((0.0, 0.0), n_points) + + for i in eachindex(mz) + # Determine the bin for the current point + # Bin indices are 1-based + bin_index = min(n_points, floor(Int, (mz[i] - mz_min) / bin_width) + 1) + + # If the current point's intensity is higher than what's in the bin, replace it + if intensity[i] > bins[bin_index][1] + bins[bin_index] = (intensity[i], mz[i]) + end + end + + # Filter out empty bins and separate the mz and intensity values + final_mz = [b[2] for b in bins if b[1] > 0.0] + final_intensity = [b[1] for b in bins if b[1] > 0.0] + + return final_mz, final_intensity +end + # ============================================================================ # # @@ -678,3 +796,97 @@ function plot_slices(slices, names, masses, output_dir; stage_name, bins=256, dp end return fig end + +""" + save_bitmap(name::String, pixMap::Matrix{UInt8}, colorTable::Vector{UInt32}) + +Saves an 8-bit indexed image as a BMP file. +This is a modernized version of `SaveBitmapCl`. +""" +function save_bitmap(name::String, pixMap::Matrix{UInt8}, colorTable::Vector{UInt32}) + # Get image dimensions + height, width = size(pixMap) + + # Normalize pixel values to stretch contrast, as in the original SaveBitmapCl + minVal, maxVal = extrema(pixMap) + if maxVal > minVal + pixMap = round.(UInt8, 255 * (pixMap .- minVal) ./ (maxVal - minVal)) + end + + # Compute row padding (each row must be a multiple of 4 bytes) + padding = (4 - (width % 4)) % 4 + + # BMP color table must have 256 entries for 8-bit images + fullColorTable = Vector{UInt32}(undef, 256) + if length(colorTable) <= 256 + fullColorTable[1:length(colorTable)] .= colorTable + fullColorTable[length(colorTable)+1:end] .= 0 + else + fullColorTable .= colorTable[1:256] + end + + # Compute file dimensions + offset = 14 + 40 + (256 * 4) # 14(file) + 40(info) + 1024(palette) = 1078 + imgBytes = height * (width + padding) + fileSize = offset + imgBytes + + open(name, "w") do stream + # === File Header (14 bytes) === + write(stream, UInt16(0x4D42)) # "BM" + write(stream, UInt32(fileSize)) + write(stream, UInt16(0)) # Reserved + write(stream, UInt16(0)) # Reserved + write(stream, UInt32(offset)) + + # === Info Header (40 bytes) === + write(stream, UInt32(40)) # Info header size + write(stream, Int32(width)) + write(stream, Int32(height)) # Positive for bottom-up storage + write(stream, UInt16(1)) # Number of color planes + write(stream, UInt16(8)) # Bits per pixel + write(stream, UInt32(0)) # Compression (BI_RGB) + write(stream, UInt32(imgBytes)) + write(stream, Int32(2835)) # Pels per meter X (~72 DPI) + write(stream, Int32(2835)) # Pels per meter Y (~72 DPI) + write(stream, UInt32(256)) # Colors in color table + write(stream, UInt32(0)) # Important colors (0 = all) + + # === Color Table === + write(stream, fullColorTable) + + # === Image Pixels (written bottom-up) === + row_buffer = Vector{UInt8}(undef, width + padding) + row_buffer[width+1:end] .= 0 # pre-fill padding bytes + + for i in height:-1:1 + row_data = @view pixMap[i, :] + row_buffer[1:width] = row_data + write(stream, row_buffer) + end + end +end + +# ******************************************************************** +# Viridis color palette (256 colors) +# ******************************************************************** + +""" + generate_palette(colorscheme, n_colors=256) + +Generates a BMP-compatible UInt32 color palette from a ColorScheme. +""" +function generate_palette(colorscheme, n_colors=256) + palette = Vector{UInt32}(undef, n_colors) + colors = get(colorscheme, range(0, 1, length=n_colors)) + for i in 1:n_colors + c = colors[i] + r = round(UInt8, c.r * 255) + g = round(UInt8, c.g * 255) + b = round(UInt8, c.b * 255) + # BMP color table format is 0x00RRGGBB, written little-endian becomes BB GG RR 00 + palette[i] = (UInt32(r) << 16) | (UInt32(g) << 8) | UInt32(b) + end + return palette +end + +const ViridisPalette = generate_palette(ColorSchemes.viridis) diff --git a/test/readme.md b/test/readme.md index bea2736..f80ee72 100644 --- a/test/readme.md +++ b/test/readme.md @@ -1,21 +1,78 @@ -to run the tests: +# JuliaMSI Test Suite -time julia --project=. test/run_tests.jl +This document provides instructions on how to set up and run the test suite for the `JuliaMSI` package. The tests validate the core functionality of the data processing workflows, including loading, converting, and visualizing mass spectrometry data. -to enable the different test cases, there are boolean variables +## Local Installation and Setup -test1 = true -test2 = true -test3 = true +### 1. Prerequisites +Make sure you have Julia installed (at least version 1.6). If not, we recommend using `juliaup` for installation. +- **juliaup**: [https://github.com/JuliaLang/juliaup](https://github.com/JuliaLang/juliaup) +- **Official binaries**: [https://julialang.org/downloads/](https://julialang.org/downloads/) -set them to true to perform that specific test. +### 2. Download the Project +Download and decompress the repository. You can get it from the official Codeberg page or clone it using Git. +- **Download ZIP**: [https://codeberg.org/LabABI/JuliaMSI/archive/main.zip](https://codeberg.org/LabABI/JuliaMSI/archive/main.zip) +- **Git URL**: `https://codeberg.org/LabABI/JuliaMSI.git` -test 1 consists in mzml validation and spectrum plotting +Ensure you know the directory where the `JuliaMSI` files are located. -test 2 consists in imzml conversion and validation +### 3. Download Test Data +The test script requires example mass spectrometry data files. You can download the required test data from Zenodo: +- **Example Data**: [https://doi.org/10.5281/zenodo.10084132](https://doi.org/10.5281/zenodo.10084132) -test 3 consists in imzml validation and spectrum and slice plotting +Download the data and place it in a known location on your computer. -use the global variables: -TEST_MZML_FILE, SPECTRUM_TO_PLOT, CONVERSION_SOURCE_MZML, CONVERSION_SYNC_FILE, CONVERSION_TARGET_IMZML, TEST_IMZML_FILE, MZ_VALUE_FOR_SLICE, MZ_TOLERANCE, COORDS_TO_PLOT, RESULTS_DIR -to modify input that the test script will recieve. +### 4. Configure the Test Script +Before running the tests, you must edit the `test/run_tests.jl` file to point to the test data you downloaded. + +1. Open `test/run_tests.jl` in a text editor. +2. Locate the `CONFIG: PLEASE FILL IN YOUR FILE PATHS HERE` section. +3. Update the constant variables (e.g., `TEST_MZML_FILE`, `CONVERSION_SOURCE_MZML`, `CONVERSION_SYNC_FILE`) with the absolute paths to the corresponding files on your system. + +## Running the Tests + +1. **Navigate to the Project Directory**: + Open your terminal and use the `cd` command to navigate to the root folder of the `JuliaMSI` project. + ```bash + cd /path/to/your/JuliaMSI + ``` + +2. **Execute the Test Script**: + Run the following command from the project's root directory. This will install the necessary dependencies and run the tests. + ```bash + julia --project=. test/run_tests.jl + ``` + +3. **Check the Results**: + The script will print its progress to the console. Any generated images (plots and image slices) will be saved in the `test/results/` directory. + +## Test Case Configuration + +You can customize the test run by editing the variables in `test/run_tests.jl`. + +### Enabling and Disabling Test Cases +You can run or skip specific test cases by setting the corresponding boolean variables to `true` or `false`. + +```julia +test1 = true # Runs Test Case 1 +test2 = true # Runs Test Case 2 +test3 = true # Runs Test Case 3 +``` + +- **Test Case 1**: Validates a standard `.mzML` file and generates plots for a single spectrum, the total spectrum, and the average spectrum. +- **Test Case 2**: Tests the conversion of a `.mzML` file (and its corresponding `.txt` sync file) into the `.imzML` format. It then validates the newly created file. +- **Test Case 3**: Validates an existing `.imzML` file. It generates plots for a spectrum at specific coordinates, the total spectrum, the average spectrum, and an ion image slice. + +### Input Configuration Variables +All configuration variables are located in the `CONFIG` section of `test/run_tests.jl`. + +- `TEST_MZML_FILE`: Path to the standard `.mzML` file for Test Case 1. +- `SPECTRUM_TO_PLOT`: The index of the spectrum to plot from the `.mzML` file in Test Case 1. +- `CONVERSION_SOURCE_MZML`: Path to the imaging `.mzML` file to be converted in Test Case 2. +- `CONVERSION_SYNC_FILE`: Path to the `.txt` synchronization file corresponding to `CONVERSION_SOURCE_MZML`. +- `CONVERSION_TARGET_IMZML`: The output path for the `.imzML` file generated in Test Case 2. This is also used as the default input for Test Case 3. +- `TEST_IMZML_FILE`: Path to the `.imzML` file to be tested in Test Case 3. +- `MZ_VALUE_FOR_SLICE`: The m/z value to use for creating an ion image slice in Test Case 3. +- `MZ_TOLERANCE`: The tolerance (+/-) for the `MZ_VALUE_FOR_SLICE` when generating the image. +- `COORDS_TO_PLOT`: An `(X, Y)` tuple specifying the coordinates of the spectrum to plot from the `.imzML` file in Test Case 3. +- `RESULTS_DIR`: The directory where output images will be saved. \ No newline at end of file diff --git a/test/run_tests.jl b/test/run_tests.jl index d96c39f..9b2efb2 100644 --- a/test/run_tests.jl +++ b/test/run_tests.jl @@ -32,7 +32,8 @@ using MSI_src # A regular, non-imaging mzML file. # const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/mzML/T9_A1.mzML" # const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" -const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" +# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" +const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.mzML" const SPECTRUM_TO_PLOT = 1 # Which spectrum to plot from the file # --- Test Case 2: .mzML + Sync File for Conversion --- @@ -49,15 +50,18 @@ const CONVERSION_TARGET_IMZML = "test/results/converted_mzml.imzML" # --- Test Case 3: Standard .imzML file --- # An existing imzML file (can be the one generated from Case 2). -const TEST_IMZML_FILE = CONVERSION_TARGET_IMZML # The output from case 2 +# const TEST_IMZML_FILE = CONVERSION_TARGET_IMZML # The output from case 2 # const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML" # const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.imzML" +const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging prueba Roya 1/royaimg.imzML" # The m/z value to use for creating an image slice. -const MZ_VALUE_FOR_SLICE = 309.06 # BF +# const MZ_VALUE_FOR_SLICE = 309.06 # BF # const MZ_VALUE_FOR_SLICE = 896.0 # HR2MSI # const MZ_VALUE_FOR_SLICE = 76.03 # I PS +const MZ_VALUE_FOR_SLICE = 473 # ROYA # const MZ_TOLERANCE = 0.1 -const MZ_TOLERANCE = 1 +# const MZ_TOLERANCE = 1 +const MZ_TOLERANCE = 0.5 # Coordinates to plot a specific spectrum from imzML const COORDS_TO_PLOT = (50, 50) # Example coordinates (X, Y) @@ -65,8 +69,8 @@ const COORDS_TO_PLOT = (50, 50) # Example coordinates (X, Y) # --- Output Directory --- const RESULTS_DIR = "test/results" -test1 = true -test2 = true +test1 = false +test2 = false test3 = true # =================================================================== @@ -216,8 +220,7 @@ function run_test() end # --- Test Case 3: Process an existing .imzML file --- - println(" -" * "="^20 * " Test Case 3: Processing .imzML " * "="^20) + println("" * "="^20 * " Test Case 3: Processing .imzML " * "="^20) if test3 == true # Run new, stronger validation for imzML validate_msi_data(TEST_IMZML_FILE) @@ -226,6 +229,7 @@ function run_test() if isfile(TEST_IMZML_FILE) # Add spectrum plotting for imzML to match Test Case 1 try + """ # Get the msi data from the imzml println("Plotting a sample spectrum from $TEST_IMZML_FILE...") msi_data = OpenMSIData(TEST_IMZML_FILE) @@ -271,6 +275,8 @@ function run_test() output_path = joinpath(RESULTS_DIR, "test_imzml_average_spectrum.png") save(output_path, fig) println("SUCCESS: Total spectrum plot saved to $output_path") + """ + println("No spectrums tested on this try.") catch e println("ERROR during spectrum plotting in Test Case 3: $e") end @@ -279,7 +285,7 @@ function run_test() try println("Testing plot_slice function on $TEST_IMZML_FILE...") msi_data = OpenMSIData(TEST_IMZML_FILE) - plot_slice(msi_data, MZ_VALUE_FOR_SLICE, MZ_TOLERANCE, RESULTS_DIR, stage_name="test_imzml_single_slice") + @time plot_slice(msi_data, MZ_VALUE_FOR_SLICE, MZ_TOLERANCE, RESULTS_DIR, stage_name="test_imzml_single_slice") # The success message is now inside plot_slice catch e println("ERROR during plot_slice test in Test Case 3: $e") From 7a0360ef68c1c40b327ecf75b4880765d6405399 Mon Sep 17 00:00:00 2001 From: Pixelguy14 Date: Mon, 6 Oct 2025 14:58:37 -0600 Subject: [PATCH 04/19] added docstrings to most functions in the src folder, altered msi data calculations for fast iterators, improved total spectrumtimes, added closer colorbar bin calculations, plus more small changes --- Manifest.toml | 2 +- Project.toml | 1 + app.jl | 29 ++-- julia_imzML_visual.jl | 60 ++++++++- src/MSIData.jl | 299 +++++++++++++++++++++++++++++++++--------- src/MSI_src.jl | 2 +- src/MzmlConverter.jl | 92 ++++++++++++- src/ParserHelpers.jl | 22 +++- src/imzML.jl | 73 ++++++++++- src/mzML.jl | 73 ++++++++++- test/run_tests.jl | 30 ++--- 11 files changed, 578 insertions(+), 105 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index e1ae255..456263d 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.11.7" manifest_format = "2.0" -project_hash = "0f3959248d174e05d23b9e9267b00c6d2ead256d" +project_hash = "d4aecbadf6a54893101b3b89b57e8ae0ab391000" [[deps.ATK_jll]] deps = ["Artifacts", "Glib_jll", "JLLWrappers", "Libdl"] diff --git a/Project.toml b/Project.toml index 0db27f1..ff9fc31 100644 --- a/Project.toml +++ b/Project.toml @@ -15,6 +15,7 @@ GenieFramework = "a59fdf5c-6bf0-4f5d-949c-a137c9e2f353" Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" Libz = "2ec943e9-cfe8-584d-b93d-64dcb6d567b7" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Mmap = "a63ad114-7e13-5084-954f-fe012c677804" NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4" NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85" PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5" diff --git a/app.jl b/app.jl index b836bce..f619549 100644 --- a/app.jl +++ b/app.jl @@ -15,6 +15,7 @@ using LinearAlgebra using NativeFileDialog # Opens the file explorer depending on the OS using StipplePlotly using Base.Filesystem: mv # To rename files in the system +using Printf # Required for @sprintf macro in colorbar generation # Bring MSIData into App module's scope using .MSI_src: MSIData, OpenMSIData, GetSpectrum, IterateSpectra, ImzMLSource, _iterate_spectra_fast, MzMLSource, find_mass, ViridisPalette, get_mz_slice, quantize_intensity, save_bitmap, median_filter, save_bitmap, downsample_spectrum, TrIQ @@ -295,16 +296,16 @@ include("./julia_imzML_visual.jl") fTime = time() eTime = round(fTime - sTime, digits=3) - msg = "File loaded in $(eTime) seconds. Calculating total spectrum..." + # msg = "File loaded in $(eTime) seconds. Calculating total spectrum..." + msg = "File loaded in $(eTime) seconds." # Enable UI controls btnStartDisable = !(msi_data.source isa ImzMLSource) btnPlotDisable = false btnSpectraDisable = false SpectraEnabled = true - + """ # --- Automatically generate and display the sum spectrum plot --- - # This is still part of the same async task progressSpectraPlot = true sTime = time() @@ -314,7 +315,7 @@ include("./julia_imzML_visual.jl") fTime = time() eTime = round(fTime - sTime, digits=3) msg = "Total spectrum plot loaded in $(eTime) seconds." - + """ catch e msi_data = nothing msg = "Error loading file: $e" @@ -324,7 +325,11 @@ include("./julia_imzML_visual.jl") SpectraEnabled = false @error "File loading failed" exception=(e, catch_backtrace()) finally - # This now correctly runs after everything is finished + # This block will always run at the end of the async task + GC.gc() + if Sys.islinux() + ccall(:malloc_trim, Int32, (Int32,), 0) + end progress = false progressSpectraPlot = false end @@ -347,7 +352,7 @@ include("./julia_imzML_visual.jl") msg = "No .imzML file loaded or selected file is not an .imzML. Please select a valid file." warning_msg = true elseif Nmass > 0 && Tol > 0 && Tol <= 1 && colorLevel > 1 && colorLevel < 257 - msg = "Creating image for Nmass=$(Nmass) Tol=$(Tol). Please be patient." + msg = "Creating image for m/z=$(Nmass) Tol=$(Tol). Please be patient." try # Use the new get_mz_slice with the centralized MSIData object slice = get_mz_slice(msi_data, Nmass, Tol) @@ -371,11 +376,8 @@ include("./julia_imzML_visual.jl") current_triq = "TrIQ_$(text_nmass).bmp" msgtriq = "TrIQ image with the Nmass of $(replace(text_nmass, "_" => "."))" - bound = MSI_src.get_outlier_thres(slice, triqProb) - levels = range(bound[1], stop=bound[2], length=8) - levels = vcat(levels, 2 * levels[end] - levels[end-1]) - Colorbar(fig[1, 1], colormap=cgrad(:viridis, colorLevel, categorical=true), limits=(0, bound[2]), ticks=levels, tickformat=log_tick_formatter, label="Intensity", size=25) - save("public/colorbar_TrIQ_$(text_nmass).png", fig) + colorbar_path = joinpath("public", "colorbar_TrIQ_$(text_nmass).png") + generate_colorbar_image(slice, colorLevel, colorbar_path, use_triq=true, triq_prob=triqProb) colorbarT = "/colorbar_TrIQ_$(text_nmass).png?t=$(timestamp)" current_col_triq = "colorbar_TrIQ_$(text_nmass).png" @@ -400,9 +402,8 @@ include("./julia_imzML_visual.jl") current_msi = "MSI_$(text_nmass).bmp" msgimg = "Image with the Nmass of $(replace(text_nmass, "_" => "."))" - levels = range(0, maximum(slice), length=8) - Colorbar(fig[1, 1], colormap=cgrad(:viridis, colorLevel, categorical=true), limits=(0, maximum(slice)), ticks=levels, tickformat=log_tick_formatter, label="Intensity", size=25) - save("public/colorbar_MSI_$(text_nmass).png", fig) + colorbar_path = joinpath("public", "colorbar_MSI_$(text_nmass).png") + generate_colorbar_image(slice, colorLevel, colorbar_path) colorbar = "/colorbar_MSI_$(text_nmass).png?t=$(timestamp)" current_col_msi = "colorbar_MSI_$(text_nmass).png" diff --git a/julia_imzML_visual.jl b/julia_imzML_visual.jl index e20abf7..666faff 100644 --- a/julia_imzML_visual.jl +++ b/julia_imzML_visual.jl @@ -325,7 +325,66 @@ function log_tick_formatter(values::Vector{Float64}) formValues[i]=value end return map((v, e) -> e == 0 ? "$(round(v, sigdigits=2))" : "$(round(v, sigdigits=2))x10" * Makie.UnicodeFun.to_superscript(e), formValues, exponents) +end +function generate_colorbar_image(slice_data::AbstractMatrix, color_levels::Int, output_path::String; use_triq::Bool=false, triq_prob::Float64=0.98) + # 1. Determine bounds based on whether TrIQ is used + min_val, max_val = if use_triq + MSI_src.get_outlier_thres(slice_data, triq_prob) + else + extrema(slice_data) + end + + # 2. Replicate the tick calculation logic from plot_slices + bins = color_levels + levels = range(min_val, stop=max_val, length=bins + 1) + level_range = levels[end] - levels[1] + + if level_range == 0 + levels = range(min_val - 0.1, stop=max_val + 0.1, length=bins + 1) + level_range = 0.2 + end + + exponent = level_range > 0 ? floor(log10(level_range)) / 3 : 0 + scale = 10^(3 * exponent) + scaled_levels = levels ./ scale + + format_num = level_range > 0 ? floor(log10(level_range)) % 3 : 0 + labels = if format_num == 0 + [ @sprintf("%3.2f", lvl) for lvl in scaled_levels] + elseif format_num == 1 + [ @sprintf("%3.2f", lvl) for lvl in scaled_levels] + else + [ @sprintf("%3.2f", lvl) for lvl in scaled_levels] + end + + divisors = 2:7 + remainders = (bins - 1) .% divisors + best_divisor = divisors[findlast(x -> x == minimum(remainders), remainders)] + tick_indices = round.(Int, range(1, stop=bins + 1, length=best_divisor + 1)) + + if !(1 in tick_indices) + pushfirst!(tick_indices, 1) + end + if !((bins + 1) in tick_indices) + push!(tick_indices, bins + 1) + end + unique!(sort!(tick_indices)) + + tick_positions = levels[tick_indices] + tick_labels = labels[tick_indices] + + # 3. Create and save the colorbar image + fig = Figure(size=(150, 250)) + Colorbar(fig[1, 1], + colormap=cgrad(:viridis, bins), + limits=(min_val, max_val), + label=(scale == 1 ? "Intensity" : "Intensity ×10^$(round(Int, 3 * exponent))"), + ticks=(tick_positions, tick_labels), + labelsize=20, + ticklabelsize=16 + ) + save(output_path, fig) end # meanSpectrumPlot recieves the local directory of the image as a string, @@ -408,7 +467,6 @@ function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int, plotdata = [trace] plotlayout = layout - # Return the full data for other uses, and the plot data return plotdata, plotlayout, mz, intensity end diff --git a/src/MSIData.jl b/src/MSIData.jl index b570ee3..1bdfb61 100644 --- a/src/MSIData.jl +++ b/src/MSIData.jl @@ -6,27 +6,50 @@ including caching and iteration logic, for handling large mzML and imzML dataset efficiently. """ -using Base64, Libz, Serialization # For reading binary data +using Base64, Libz, Serialization, Printf # For reading binary data # Abstract type for different data sources (e.g., mzML, imzML) # This allows dispatching to the correct binary reading logic. abstract type MSDataSource end # Concrete type for imzML data sources +""" + ImzMLSource <: MSDataSource + +A data source for `.imzML` files, holding a handle to the binary `.ibd` file +and the expected format for m/z and intensity arrays. +""" struct ImzMLSource <: MSDataSource ibd_handle::IO mz_format::Type intensity_format::Type end -# Concrete type for mzML data sources +""" + MzMLSource <: MSDataSource + +A data source for `.mzML` files, holding a handle to the `.mzML` file itself +(which contains the binary data encoded in Base64) and the expected data formats. +""" struct MzMLSource <: MSDataSource file_handle::IO mz_format::Type intensity_format::Type end -# Struct to hold info about a binary data array (mz or intensity) +""" + SpectrumAsset + +Contains metadata for a single binary data array (m/z or intensity) within a spectrum. + +# Fields +- `format`: The data type of the elements (e.g., `Float32`, `Int64`). +- `is_compressed`: A boolean flag indicating if the data is compressed (e.g., with zlib). +- `offset`: The byte offset of the data within the file (`.ibd` for imzML, `.mzML` for mzML). +- `encoded_length`: The length of the data. For mzML, this is the Base64 encoded length. + For imzML, this is the number of elements in the array. +- `axis_type`: A symbol (`:mz` or `:intensity`) indicating the type of data. +""" struct SpectrumAsset format::Type is_compressed::Bool @@ -37,7 +60,17 @@ struct SpectrumAsset axis_type::Symbol end -# Metadata for a single spectrum, common to both formats +""" + SpectrumMetadata + +Contains all metadata for a single spectrum, common to both imzML and mzML formats. + +# Fields +- `x`, `y`: The spatial coordinates of the spectrum (for imzML only). +- `id`: The unique identifier string for the spectrum (for mzML only). +- `mz_asset`: A `SpectrumAsset` for the m/z array. +- `int_asset`: A `SpectrumAsset` for the intensity array. +""" struct SpectrumMetadata # For imzML x::Int32 @@ -51,7 +84,22 @@ struct SpectrumMetadata int_asset::SpectrumAsset end -# The main unified data object +""" + MSIData + +The primary object for interacting with mass spectrometry data. It provides a unified +interface for both `.mzML` and `.imzML` files and includes an LRU cache for +efficient repeated access to spectra. + +# Fields +- `source`: The underlying `MSDataSource` (`ImzMLSource` or `MzMLSource`). +- `spectra_metadata`: A vector of `SpectrumMetadata` for all spectra in the file. +- `image_dims`: A tuple `(width, height)` of the spatial dimensions (for imzML). +- `coordinate_map`: A matrix mapping `(x, y)` coordinates to a linear spectrum index (for imzML). +- `cache`: A dictionary holding cached spectra. +- `cache_order`: A vector tracking the usage order for the LRU cache. +- `cache_size`: The maximum number of spectra to store in the cache. +""" mutable struct MSIData source::MSDataSource spectra_metadata::Vector{SpectrumMetadata} @@ -81,6 +129,26 @@ end # --- Internal function for reading binary data --- # +""" + read_binary_vector(io::IO, asset::SpectrumAsset) + +Reads and decodes a single binary data vector (like m/z or intensity array) +from a `.mzML` file. The data is expected to be Base64-encoded and may be +compressed. + +This internal function handles: +1. Reading the raw Base64 string. +2. Decoding from Base64. +3. Decompressing the data if `asset.is_compressed` is true. +4. Converting the byte order from network (big-endian) to host order. + +# Arguments +- `io`: The IO stream of the `.mzML` file. +- `asset`: The `SpectrumAsset` containing metadata for the array. + +# Returns +- A `Vector` of the appropriate type containing the decoded data. +""" function read_binary_vector(io::IO, asset::SpectrumAsset) seek(io, asset.offset) raw_b64 = read(io, asset.encoded_length) @@ -94,6 +162,20 @@ function read_binary_vector(io::IO, asset::SpectrumAsset) end # Overload for different source types +""" + read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata) + +Reads a single spectrum's m/z and intensity arrays directly from the `.ibd` +binary file for an `.imzML` dataset. It handles reading the raw binary data +and converting it from little-endian to the host's native byte order. + +# Arguments +- `source`: An `ImzMLSource` containing the file handle and data formats. +- `meta`: The `SpectrumMetadata` for the spectrum to be read. + +# Returns +- A tuple `(mz, intensity)` containing the two requested data arrays. +""" function read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata) # For imzML, the binary data is raw, not base64 encoded. # The `encoded_length` field in this case holds the number of points. @@ -106,9 +188,27 @@ function read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata) seek(source.ibd_handle, meta.int_asset.offset) read!(source.ibd_handle, intensity) + # imzML data is little-endian. Convert to host byte order. + mz .= ltoh.(mz) + intensity .= ltoh.(intensity) + return mz, intensity end +""" + read_spectrum_from_disk(source::MzMLSource, meta::SpectrumMetadata) + +Reads a single spectrum's m/z and intensity arrays from a `.mzML` file. +This is a high-level function that orchestrates the reading process by calling +`read_binary_vector` for each of the m/z and intensity arrays. + +# Arguments +- `source`: A `MzMLSource` containing the file handle. +- `meta`: The `SpectrumMetadata` for the spectrum to be read. + +# Returns +- A tuple `(mz, intensity)` containing the two requested data arrays. +""" function read_spectrum_from_disk(source::MzMLSource, meta::SpectrumMetadata) # For mzML, data is Base64 encoded within the XML mz = read_binary_vector(source.file_handle, meta.mz_asset) @@ -182,13 +282,14 @@ end using Serialization function get_total_spectrum_imzml(msi_data::MSIData; num_bins::Int=2000) - println("Calculating total spectrum (2-pass method for imzML)...") + println("Calculating total spectrum for imzML (2-pass method)...") + total_start_time = time_ns() # 1. First Pass: Find the global m/z range + pass1_start_time = time_ns() println(" Pass 1: Finding global m/z range...") global_min_mz = Inf global_max_mz = -Inf - _iterate_spectra_fast(msi_data) do idx, mz, _ if !isempty(mz) local_min, local_max = extrema(mz) @@ -196,36 +297,66 @@ function get_total_spectrum_imzml(msi_data::MSIData; num_bins::Int=2000) global_max_mz = max(global_max_mz, local_max) end end + pass1_duration = (time_ns() - pass1_start_time) / 1e9 if !isfinite(global_min_mz) - @warn "Could not determine a valid m/z range. All spectra might be empty." + @warn "Could not determine a valid m/z range for imzML. All spectra might be empty." return (Float64[], Float64[]) end println(" Global m/z range found: [$(global_min_mz), $(global_max_mz)]") - # 2. Define Bins + # 2. Define Bins and precompute constants mz_bins = range(global_min_mz, stop=global_max_mz, length=num_bins) intensity_sum = zeros(Float64, num_bins) bin_step = step(mz_bins) + inv_bin_step = 1.0 / bin_step # Precompute reciprocal to avoid division + min_mz = global_min_mz - # 3. Second Pass: Sum intensities into bins + # 3. Second Pass: Optimized binning + pass2_start_time = time_ns() println(" Pass 2: Summing intensities into $num_bins bins...") _iterate_spectra_fast(msi_data) do idx, mz, intensity if isempty(mz) - return # continue equivalent + return end - for i in eachindex(mz) - bin_index = clamp(round(Int, (mz[i] - global_min_mz) / bin_step) + 1, 1, num_bins) - intensity_sum[bin_index] += intensity[i] + + # Use views to avoid bounds checks on the input arrays + mz_view = mz + intensity_view = intensity + + # Manual binning without clamp/round for SIMD + @inbounds for i in eachindex(mz_view) + # Calculate raw bin index (much faster than round+clamp) + raw_index = (mz_view[i] - min_mz) * inv_bin_step + 1.0 + bin_index = trunc(Int, raw_index) + + # Manual bounds checking (faster than clamp) + if 1 <= bin_index <= num_bins + intensity_sum[bin_index] += intensity_view[i] + elseif bin_index < 1 + intensity_sum[1] += intensity_view[i] + else # bin_index > num_bins + intensity_sum[num_bins] += intensity_view[i] + end end end - - println("Total spectrum calculation complete.") + pass2_duration = (time_ns() - pass2_start_time) / 1e9 + + total_duration = (time_ns() - total_start_time) / 1e9 + println("\n--- imzML Profiling ---") + @printf " Pass 1 (I/O only): %.2f seconds\n" pass1_duration + @printf " Pass 2 (I/O + Binning): %.2f seconds\n" pass2_duration + @printf " Est. Binning Overhead: %.2f seconds\n" (pass2_duration - pass1_duration) + @printf " Total Function Time: %.2f seconds\n" total_duration + println("-------------------------\n") + + println("Total spectrum calculation complete for imzML.") return (collect(mz_bins), intensity_sum) end function get_total_spectrum_mzml(msi_data::MSIData; num_bins::Int=2000) - println("Calculating total spectrum (single-pass optimization for mzML)...") + println("Calculating total spectrum for mzML (optimized single-pass method)...") + total_start_time = time_ns() num_spectra = length(msi_data.spectra_metadata) if num_spectra == 0 return (Float64[], Float64[]) @@ -233,31 +364,33 @@ function get_total_spectrum_mzml(msi_data::MSIData; num_bins::Int=2000) temp_path, temp_io = mktemp() try - # --- Pass 1: Write spectra to temp file and find min/max m/z --- - println(" Pass 1: Caching spectra and finding global m/z range...") + # --- Pass 1: Read from source, find m/z range, and write decoded spectra to temp file --- + pass1_start_time = time_ns() + println(" Pass 1: Caching decoded spectra and finding global m/z range...") global_min_mz = Inf global_max_mz = -Inf - + _iterate_spectra_fast(msi_data) do idx, mz, intensity - Serialization.serialize(temp_io, (mz, intensity)) if !isempty(mz) local_min, local_max = extrema(mz) global_min_mz = min(global_min_mz, local_min) global_max_mz = max(global_max_mz, local_max) end + Serialization.serialize(temp_io, (mz, intensity)) end flush(temp_io) + pass1_duration = (time_ns() - pass1_start_time) / 1e9 if !isfinite(global_min_mz) - @warn "Could not determine a valid m/z range. All spectra might be empty." + @warn "Could not determine a valid m/z range for mzML. All spectra might be empty." return (Float64[], Float64[]) end println(" Global m/z range found: [$(global_min_mz), $(global_max_mz)]") - # --- Pass 2: Read from temp file and bin intensities --- + # --- Pass 2: Read from fast temp file and bin intensities --- + pass2_start_time = time_ns() println(" Pass 2: Reading from cache and summing intensities into $num_bins bins...") - seekstart(temp_io) mz_bins = range(global_min_mz, stop=global_max_mz, length=num_bins) intensity_sum = zeros(Float64, num_bins) @@ -265,7 +398,6 @@ function get_total_spectrum_mzml(msi_data::MSIData; num_bins::Int=2000) while !eof(temp_io) mz, intensity = Serialization.deserialize(temp_io)::Tuple{AbstractVector, AbstractVector} - if isempty(mz) continue end @@ -274,18 +406,27 @@ function get_total_spectrum_mzml(msi_data::MSIData; num_bins::Int=2000) intensity_sum[bin_index] += intensity[i] end end - - println("Total spectrum calculation complete.") + pass2_duration = (time_ns() - pass2_start_time) / 1e9 + + total_duration = (time_ns() - total_start_time) / 1e9 + println("\n--- mzML Profiling ---") + @printf " Pass 1 (Read+Decode+Cache): %.2f seconds\n" pass1_duration + @printf " Pass 2 (Read Cache+Bin): %.2f seconds\n" pass2_duration + @printf " Total Function Time: %.2f seconds\n" total_duration + println("----------------------\n") + + println("Total spectrum calculation complete for mzML.") return (collect(mz_bins), intensity_sum) finally close(temp_io) rm(temp_path, force=true) + println(" Temporary cache file removed.") end end """ - get_total_spectrum(msi_data::MSIData; num_bins::Int=20000) -> Tuple{Vector{Float64}, Vector{Float64}} + get_total_spectrum(msi_data::MSIData; num_bins::Int=2000) -> Tuple{Vector{Float64}, Vector{Float64}} Calculates the sum of all spectra in the dataset by binning. This function dispatches to a specialized implementation based on the file type @@ -301,7 +442,6 @@ function get_total_spectrum(msi_data::MSIData; num_bins::Int=2000) end end - """ get_average_spectrum(msi_data::MSIData; num_bins::Int=20000) -> Tuple{Vector{Float64}, Vector{Float64}} @@ -328,6 +468,12 @@ end # --- Iterator Implementation --- # +""" + MSIDataIterator + +A stateful iterator for sequentially accessing spectra from an `MSIData` object. +This is created by the `IterateSpectra` function. +""" struct MSIDataIterator data::MSIData end @@ -336,7 +482,7 @@ end IterateSpectra(data::MSIData) Returns an iterator that yields each spectrum, processing the file sequentially -with minimal memory overhead. +with minimal memory overhead. This iterator supports caching via `GetSpectrum`. This function is the core of the "Event-driven" access pattern. """ @@ -348,6 +494,20 @@ end Base.length(it::MSIDataIterator) = length(it.data.spectra_metadata) Base.eltype(it::MSIDataIterator) = Tuple{Int, Tuple{Vector, Vector}} # Index, (mz, intensity) +""" + Base.iterate(it::MSIDataIterator, state=1) + +Advances the iterator, returning the next spectrum in the sequence. It calls +`GetSpectrum`, which means the iteration process is cached according to the +`MSIData` object's cache settings. + +# Arguments +- `it`: The `MSIDataIterator` instance. +- `state`: The index of the next spectrum to retrieve. + +# Returns +- A tuple `((index, spectrum), next_state)` or `nothing` if iteration is complete. +""" function Base.iterate(it::MSIDataIterator, state=1) if state > length(it.data.spectra_metadata) return nothing # End of iteration @@ -364,35 +524,14 @@ end # --- High-performance Internal Iterator --- # """ - _iterate_spectra_fast(f::Function, data::MSIData) + _iterate_spectra_fast_impl(f::Function, data::MSIData, source::ImzMLSource) -An internal, high-performance iterator for bulk processing. -It avoids the overhead of `GetSpectrum` by reading directly from the file stream -and reusing pre-allocated buffers. - -- `f`: A function to call for each spectrum, with signature `f(index, mz, intensity)`. -- `data`: The `MSIData` object. +Internal implementation of the fast iterator for `.imzML` files. It reads +data directly from the `.ibd` file stream and reuses pre-allocated buffers +to minimize memory allocations and overhead, making it ideal for bulk processing tasks. """ function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::ImzMLSource) - # This implementation is for imzML and is optimized for performance by - # using pre-allocated buffers. - max_points = 0 - for meta in data.spectra_metadata - # For imzML, encoded_length is the number of points - max_points = max(max_points, meta.mz_asset.encoded_length) - end - - # If all spectra are empty, just call f with empty arrays - if max_points == 0 && !isempty(data.spectra_metadata) - for i in 1:length(data.spectra_metadata) - f(i, source.mz_format[], source.intensity_format[]) - end - return - end - - mz_buffer = Vector{source.mz_format}(undef, max_points) - int_buffer = Vector{source.intensity_format}(undef, max_points) - + # Optimized implementation: allocates arrays per spectrum and uses an efficient I/O pattern. for i in 1:length(data.spectra_metadata) meta = data.spectra_metadata[i] nPoints = meta.mz_asset.encoded_length @@ -402,19 +541,43 @@ function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::ImzMLSou continue end - mz_view = view(mz_buffer, 1:nPoints) - int_view = view(int_buffer, 1:nPoints) + # Allocate fresh arrays for each spectrum. + mz = Vector{source.mz_format}(undef, nPoints) + intensity = Vector{source.intensity_format}(undef, nPoints) - seek(source.ibd_handle, meta.mz_asset.offset) - read!(source.ibd_handle, mz_view) + # --- I/O OPTIMIZATION: Seek only once per spectrum --- + # The mz and intensity data are contiguous, so after reading the first, + # we can immediately read the second without a costly second seek. + if meta.mz_asset.offset < meta.int_asset.offset + # m/z is first, seek to it. + seek(source.ibd_handle, meta.mz_asset.offset) + # Read m/z, then immediately read intensity. + read!(source.ibd_handle, mz) + read!(source.ibd_handle, intensity) + else + # intensity is first, seek to it. + seek(source.ibd_handle, meta.int_asset.offset) + # Read intensity, then immediately read m/z. + read!(source.ibd_handle, intensity) + read!(source.ibd_handle, mz) + end - seek(source.ibd_handle, meta.int_asset.offset) - read!(source.ibd_handle, int_view) + # Convert byte order. + mz .= ltoh.(mz) + intensity .= ltoh.(intensity) - f(i, mz_view, int_view) + f(i, mz, intensity) end end +""" + _iterate_spectra_fast_impl(f::Function, data::MSIData, source::MzMLSource) + +Internal implementation of the fast iterator for `.mzML` files. It iterates +through each spectrum, decodes the Base64 data on the fly, and calls the +provided function. It bypasses the `GetSpectrum` cache to avoid storing +all decoded spectra in memory. +""" function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::MzMLSource) # This implementation is for mzML. It iterates through each spectrum, # decodes it, and then calls the function `f`. It's less performant than @@ -430,6 +593,16 @@ function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::MzMLSour end end +""" + _iterate_spectra_fast(f::Function, data::MSIData) + +An internal, high-performance iterator for bulk processing that bypasses the cache. +It dispatches to a specialized implementation based on the data source type +(`ImzMLSource` or `MzMLSource`). + +- `f`: A function to call for each spectrum, with signature `f(index, mz, intensity)`. +- `data`: The `MSIData` object. +""" function _iterate_spectra_fast(f::Function, data::MSIData) # Dispatch to the correct implementation based on the source type _iterate_spectra_fast_impl(f, data, data.source) diff --git a/src/MSI_src.jl b/src/MSI_src.jl index c6f3ce0..59a40c4 100644 --- a/src/MSI_src.jl +++ b/src/MSI_src.jl @@ -20,7 +20,7 @@ Opens a .mzML or .imzML file and prepares it for data access. This is the main entry point for the new data access API. """ -function OpenMSIData(filepath::String; cache_size=100) +function OpenMSIData(filepath::String; cache_size=300) if endswith(lowercase(filepath), ".mzml") return load_mzml_lazy(filepath, cache_size=cache_size) elseif endswith(lowercase(filepath), ".imzml") diff --git a/src/MzmlConverter.jl b/src/MzmlConverter.jl index 695eed4..f50ec52 100644 --- a/src/MzmlConverter.jl +++ b/src/MzmlConverter.jl @@ -11,13 +11,33 @@ using DataFrames, Printf, CSV # A struct to hold the processed pixel data before exporting +""" + ProcessedPixel + +A temporary struct to hold the data for a single, fully rendered pixel before +it is written to the final `.ibd` file. + +# Fields +- `coords`: A tuple `(x, y)` of the pixel's spatial coordinates. +- `mz`: The m/z array for the pixel. +- `intensity`: The calculated intensity array for the pixel. +""" struct ProcessedPixel coords::Tuple{Int, Int} mz::Vector{Float64} # Assuming m/z is consistent, can be optimized later intensity::Vector{Float32} end -# This struct will hold the metadata needed for the XML file +""" + BinaryMetadata + +A struct to hold the file offset and length for a spectrum's binary data arrays +(m/z and intensity) after they have been written to the `.ibd` file. + +# Fields +- `mz_offset`, `mz_length`: Byte offset and length for the m/z array. +- `int_offset`, `int_length`: Byte offset and length for the intensity array. +""" struct BinaryMetadata mz_offset::UInt64 mz_length::UInt64 @@ -28,8 +48,15 @@ end """ GetMzmlScanTime_linebyline(fileName::String) -(Internal) Slow, line-by-line parser for scan times. Used as a fallback if -the .mzML file index is missing or corrupt. +Parses a `.mzML` file line-by-line to extract the scan start time for each +spectrum. This is a slow and memory-intensive fallback method used only when +the faster, index-based `GetMzmlScanTime` fails. + +# Arguments +- `fileName`: Path to the `.mzML` file. + +# Returns +- A `Matrix{Int64}` where each row is `[spectrum_index, time_in_milliseconds]`. """ function GetMzmlScanTime_linebyline(fileName::String) times = Tuple{Int64, Int64}[] @@ -350,6 +377,28 @@ function MatchAcquireTime(sync_file_path::String, scans::Matrix{Int64}; img_widt end +""" + RenderPixel(pixel_info, scans, msi_data, scan_time_deltas, pixel_time_deltas) + +Reconstructs the spectrum for a single pixel by combining intensities from the +raw MS scans that occurred during the pixel's acquisition time. It uses a +weighted interpolation scheme based on the relative timing of scans and pixels. + +This function handles three cases: +1. A single scan falls entirely within the pixel's time window. +2. The pixel's time window is covered by two partial scans. +3. The pixel's time window covers one or more full scans plus two partial scans. + +# Arguments +- `pixel_info`: A row from the timing matrix containing the pixel's time and scan indices. +- `scans`: The matrix of scan times. +- `msi_data`: The `MSIData` object for the source `.mzML` file. +- `scan_time_deltas`: Pre-calculated time durations for each scan. +- `pixel_time_deltas`: Pre-calculated time durations for each pixel. + +# Returns +- A tuple `(mz_array, intensity_array)` for the rendered pixel spectrum. +""" function RenderPixel(pixel_info, scans, msi_data::MSIData, scan_time_deltas, pixel_time_deltas) pixel_time = pixel_info[3] first_scan = pixel_info[4] @@ -408,6 +457,26 @@ function RenderPixel(pixel_info, scans, msi_data::MSIData, scan_time_deltas, pix return (mz_array, new_intensity) end +""" + ConvertMzmlToImzml(source_file, target_ibd_file, timing_matrix, scans) + +Orchestrates the conversion of spectra from a `.mzML` file into a binary `.ibd` +file. It iterates through each pixel defined in the `timing_matrix`, calls +`RenderPixel` to reconstruct the pixel's spectrum, and writes the resulting +m/z and intensity arrays to the `.ibd` file in little-endian byte order. + +# Arguments +- `source_file`: Path to the source `.mzML` file. +- `target_ibd_file`: Path for the output `.ibd` binary file. +- `timing_matrix`: The output from `MatchAcquireTime`, mapping pixels to scans. +- `scans`: The matrix of scan times from `GetMzmlScanTime`. + +# Returns +- A tuple `(binary_meta_vec, coords_vec, (width, height))` containing: + - A vector of `BinaryMetadata` for each spectrum. + - A vector of `(x, y)` coordinate tuples. + - A tuple of the final image dimensions. +""" function ConvertMzmlToImzml(source_file::String, target_ibd_file::String, timing_matrix::Matrix{Int64}, scans::Matrix{Int64}) if size(timing_matrix, 1) == 0 # Create an empty .ibd file if there's nothing to process @@ -489,6 +558,23 @@ function ConvertMzmlToImzml(source_file::String, target_ibd_file::String, timing return binary_meta_vec, coords_vec, (width, height) end +""" + ExportImzml(target_file, binary_meta, coords, dims) + +Generates the `.imzML` metadata file. This XML file contains all the necessary +metadata to interpret the corresponding `.ibd` binary file, including references +to external data offsets, image dimensions, and CV parameters describing the +experiment and data format. + +# Arguments +- `target_file`: The path for the output `.imzML` file. +- `binary_meta`: A vector of `BinaryMetadata` structs with offset and length info. +- `coords`: A vector of `(x, y)` coordinates for each spectrum. +- `dims`: A tuple `(width, height)` of the final image dimensions. + +# Returns +- `true` on success, `false` on failure. +""" function ExportImzml(target_file::String, binary_meta::Vector{BinaryMetadata}, coords::Vector{Tuple{Int, Int}}, dims::Tuple{Int, Int}) ibd_file = replace(target_file, r"\.imzML$"i => ".ibd") diff --git a/src/ParserHelpers.jl b/src/ParserHelpers.jl index e0b21d6..5f52b95 100644 --- a/src/ParserHelpers.jl +++ b/src/ParserHelpers.jl @@ -59,7 +59,15 @@ end """ configure_spec_dim(stream) -Fills a `SpecDim` struct by parsing `cvParam` tags from the stream. +Parses a block of `` tags from an XML stream to configure a `SpecDim` +struct. It reads accessions to determine the data format (e.g., `Float32`), +compression status (`zlib`), and axis type (m/z vs. intensity). + +# Arguments +- `stream`: An IO stream positioned at the start of the `cvParam` block. + +# Returns +- A `SpecDim` struct populated with the parsed configuration. """ function configure_spec_dim(stream) axis = SpecDim(Float64, false, 1, 0) @@ -102,6 +110,18 @@ end # # ============================================================================ +""" + CVParams + +A mutable struct used during the parsing of mzML files to temporarily hold +CV parameter information for a data array before it is converted into a +`SpectrumAsset`. + +# Fields +- `format`: The data type (e.g., `Float64`). +- `is_compressed`: A boolean indicating if the data is compressed. +- `axis_type`: A symbol indicating the array type (`:mz` or `:intensity`). +""" mutable struct CVParams format::Type is_compressed::Bool diff --git a/src/imzML.jl b/src/imzML.jl index 0426109..61f60c3 100644 --- a/src/imzML.jl +++ b/src/imzML.jl @@ -1,3 +1,4 @@ +# src/imzML.jl using Images, Statistics, CairoMakie, DataFrames, Printf, ColorSchemes, StatsBase # --- Extracted from imzML.jl --- @@ -171,7 +172,10 @@ function load_imzml_lazy(file_path::String; cache_size=100) int_config_idx = findfirst(a -> a.Axis == 2, axis) mz_format = axis[mz_config_idx].Format intensity_format = axis[int_config_idx].Format + mz_is_compressed = axis[mz_config_idx].Packed + int_is_compressed = axis[int_config_idx].Packed println("DEBUG: m/z format: $mz_format, Intensity format: $intensity_format") + println("DEBUG: m/z compressed: $mz_is_compressed, Intensity compressed: $int_is_compressed") # --- NEW PARSING LOGIC based on the old, working code --- println("DEBUG: Learning file structure from first spectrum...") @@ -212,8 +216,8 @@ function load_imzml_lazy(file_path::String; cache_size=100) end # Create modern SpectrumAsset objects - mz_asset = SpectrumAsset(mz_format, false, mz_offset, nPoints, :mz) - int_asset = SpectrumAsset(intensity_format, false, int_offset, nPoints, :intensity) + mz_asset = SpectrumAsset(mz_format, mz_is_compressed, mz_offset, nPoints, :mz) + int_asset = SpectrumAsset(intensity_format, int_is_compressed, int_offset, nPoints, :intensity) spectra_metadata[k] = SpectrumMetadata(x, y, "", mz_asset, int_asset) @@ -449,6 +453,21 @@ end # # ============================================================================ +""" + get_outlier_thres(img, prob=0.98) + +Calculates dynamic intensity range bounds for an image based on a cumulative +probability histogram. This function replicates an R algorithm to find an +intensity threshold that corresponds to a given cumulative probability, which +is used to exclude outliers before normalization. + +# Arguments +- `img`: The input image matrix. +- `prob`: The cumulative probability threshold (default: 0.98) for outlier detection. + +# Returns +- A tuple `(low, high)` representing the calculated lower and upper intensity bounds. +""" function get_outlier_thres(img, prob=0.98) # DO NOT filter zeros. Use all pixel values like R does. int_values = vec(img) @@ -493,6 +512,21 @@ function get_outlier_thres(img, prob=0.98) return (low, actual_threshold) end +""" + set_pixel_depth(img, bounds, depth) + +Quantizes the intensity values of an image into a specified number of bins +(`depth`) within a given intensity range (`bounds`). Pixels outside the bounds +are clipped. + +# Arguments +- `img`: The input image matrix. +- `bounds`: A tuple `(min, max)` specifying the intensity range for quantization. +- `depth`: The number of quantization levels (bins). + +# Returns +- A `Matrix{UInt8}` with pixel values quantized to the specified depth. +""" function set_pixel_depth(img, bounds, depth) min_val, max_val = bounds bins = depth - 1 @@ -595,6 +629,19 @@ function quantize_intensity(slice::AbstractMatrix{<:Real}, levels::Integer=256) return image end +""" + median_filter(img) + +Applies a 3x3 median filter to the input image. This is a simple noise +reduction technique that replaces each pixel's value with the median value of +its 3x3 neighborhood. + +# Arguments +- `img`: The input image matrix. + +# Returns +- A new matrix containing the filtered image. +""" function median_filter(img) # 3x3 median filter implementation return mapwindow(median, img, (3, 3)) @@ -685,6 +732,28 @@ function display_statistics(slices, names, masses) return all_dfs end +""" + plot_slices(slices, names, masses, output_dir; stage_name, bins=256, dpi=150, global_bounds=nothing) + +Generates and saves a grid of image slice plots. Each row corresponds to a +file and each column to a mass, creating a comprehensive overview. + +# Arguments +- `slices`: A 2D array of image slice matrices (`n_files` x `n_masses`). +- `names`: A vector of file names, used for titling rows. +- `masses`: A vector of m/z values, used for titling columns. +- `output_dir`: The directory where the output plots will be saved. + +# Keyword Arguments +- `stage_name`: A string used to name the output files (e.g., "raw", "normalized"). +- `bins`: The number of color levels in the heatmap palette. +- `dpi`: The resolution for the saved image files. +- `global_bounds`: A vector of `(min, max)` tuples, one for each mass, to ensure a consistent + color scale across all files for a given mass. If `nothing`, bounds are calculated automatically. + +# Returns +- The generated `Figure` object from Makie. +""" function plot_slices(slices, names, masses, output_dir; stage_name, bins=256, dpi=150, global_bounds=nothing) n_files, n_masses = size(slices) diff --git a/src/mzML.jl b/src/mzML.jl index d65395a..90ad8f2 100644 --- a/src/mzML.jl +++ b/src/mzML.jl @@ -1,9 +1,22 @@ -# /home/pixel/Documents/Cinvestav_2025/JuliaMSI/src/mzML.jl +# src/mzML.jl # This file is responsible for parsing metadata from .mzML files. # It has been refactored to produce a unified MSIData object. -# This function is kept internal to the mzML parsing process +""" + get_spectrum_asset_metadata(stream) + +Parses a `` block within an mzML file to extract metadata +for a single data array (e.g., m/z or intensity). It reads CV parameters to +determine the data type, compression, and axis type. + +# Arguments +- `stream`: An IO stream positioned at the beginning of a `` block. + +# Returns +- A `SpectrumAsset` struct containing the parsed metadata, including the binary + data offset, encoded length, format, and compression status. +""" function get_spectrum_asset_metadata(stream) start_pos = position(stream) @@ -56,6 +69,10 @@ function get_spectrum_asset_metadata(stream) elseif acc_str == "MS:1000523" data_format = Float64 # println("DEBUG: Set format to Float64") + # MS:1000572 is a parent term for compression. While not ideal, if present, assume compression. + elseif acc_str == "MS:1000572" + compression_flag = true + # println("DEBUG: Set is_compressed to true based on parent term") elseif acc_str == "MS:1000574" compression_flag = true # println("DEBUG: Set is_compressed to true") @@ -80,6 +97,20 @@ function get_spectrum_asset_metadata(stream) end # This function is updated to return the generic SpectrumMetadata struct +""" + parse_spectrum_metadata(stream, offset::Int64) + +Parses an entire `` block from an mzML file, given a starting offset. +It extracts the spectrum ID and calls `get_spectrum_asset_metadata` to parse +the m/z and intensity array metadata. + +# Arguments +- `stream`: An IO stream for the mzML file. +- `offset`: The byte offset where the `` block begins. + +# Returns +- A `SpectrumMetadata` struct containing the parsed metadata for one spectrum. +""" function parse_spectrum_metadata(stream, offset::Int64) seek(stream, offset) @@ -97,6 +128,18 @@ function parse_spectrum_metadata(stream, offset::Int64) return SpectrumMetadata(0, 0, id, mz_asset, int_asset) end +""" + parse_offset_list(stream) + +Parses the `` block in an indexed mzML file to extract +the byte offsets for each spectrum. + +# Arguments +- `stream`: An IO stream positioned at the start of the `` block. + +# Returns +- A `Vector{Int64}` containing the byte offsets for all spectra. +""" function parse_offset_list(stream) offsets = Int64[] offset_regex = r"]*>(\d+)" @@ -121,6 +164,20 @@ function parse_offset_list(stream) end # This is the main lazy-loading function for mzML, now returning an MSIData object. +""" + load_mzml_lazy(file_path::String; cache_size::Int=100) + +Lazily loads an indexed `.mzML` file by parsing only the metadata. It reads the +spectrum index from the end of the file to get the offsets of each spectrum, +then parses the metadata for each spectrum without loading the binary data. + +# Arguments +- `file_path`: The path to the `.mzML` file. +- `cache_size`: The number of spectra to hold in an LRU cache for faster access. + +# Returns +- An `MSIData` object ready for lazy data access. +""" function load_mzml_lazy(file_path::String; cache_size::Int=100) println("DEBUG: Opening file stream for $file_path") stream = open(file_path, "r") @@ -181,8 +238,16 @@ end """ LoadMzml(fileName::String) -Eagerly loads all spectra from a .mzML file. -Provided for backward compatibility. Now uses the new MSIData architecture. +Eagerly loads all spectra from a .mzML file into memory. +This function now uses the new lazy-loading +architecture internally but presents the data in the old format. + +# Arguments +- `fileName`: The path to the `.mzML` file. + +# Returns +- A `2xN` matrix where `N` is the number of spectra. The first row contains + m/z arrays and the second row contains intensity arrays. """ function LoadMzml(fileName::String) # Use the lazy loader to get the MSIData object diff --git a/test/run_tests.jl b/test/run_tests.jl index 9b2efb2..d8c9de9 100644 --- a/test/run_tests.jl +++ b/test/run_tests.jl @@ -31,16 +31,18 @@ using MSI_src # --- Test Case 1: Standard .mzML file --- # A regular, non-imaging mzML file. # const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/mzML/T9_A1.mzML" -# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" -# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" -const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.mzML" +const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" +# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.mzML" +# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging prueba Roya 1/Roya.mzML" const SPECTRUM_TO_PLOT = 1 # Which spectrum to plot from the file # --- Test Case 2: .mzML + Sync File for Conversion --- # The special .mzML file with one spectrum per pixel. -const CONVERSION_SOURCE_MZML = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" +# const CONVERSION_SOURCE_MZML = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML" +const CONVERSION_SOURCE_MZML = TEST_MZML_FILE # The corresponding synchronization text file. const CONVERSION_SYNC_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.txt" +# const CONVERSION_SYNC_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging prueba Roya 1/Synchro.txt" # const CONVERSION_SOURCE_MZML = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.mzML" # const CONVERSION_SYNC_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.txt" @@ -50,18 +52,18 @@ const CONVERSION_TARGET_IMZML = "test/results/converted_mzml.imzML" # --- Test Case 3: Standard .imzML file --- # An existing imzML file (can be the one generated from Case 2). -# const TEST_IMZML_FILE = CONVERSION_TARGET_IMZML # The output from case 2 +const TEST_IMZML_FILE = CONVERSION_TARGET_IMZML # The output from case 2 # const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML" # const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.imzML" -const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging prueba Roya 1/royaimg.imzML" +# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging prueba Roya 1/royaimg.imzML" # The m/z value to use for creating an image slice. # const MZ_VALUE_FOR_SLICE = 309.06 # BF # const MZ_VALUE_FOR_SLICE = 896.0 # HR2MSI # const MZ_VALUE_FOR_SLICE = 76.03 # I PS -const MZ_VALUE_FOR_SLICE = 473 # ROYA +const MZ_VALUE_FOR_SLICE = 313 # ROYA # const MZ_TOLERANCE = 0.1 # const MZ_TOLERANCE = 1 -const MZ_TOLERANCE = 0.5 +const MZ_TOLERANCE = 0.1 # Coordinates to plot a specific spectrum from imzML const COORDS_TO_PLOT = (50, 50) # Example coordinates (X, Y) @@ -69,8 +71,8 @@ const COORDS_TO_PLOT = (50, 50) # Example coordinates (X, Y) # --- Output Directory --- const RESULTS_DIR = "test/results" -test1 = false -test2 = false +test1 = true +test2 = true test3 = true # =================================================================== @@ -152,7 +154,7 @@ function run_test() try # Get the msi data from the mzml println("Plotting a sample spectrum from $TEST_MZML_FILE...") - msi_data = OpenMSIData(TEST_MZML_FILE) + msi_data = @time OpenMSIData(TEST_MZML_FILE) mz, intensity = GetSpectrum(msi_data, SPECTRUM_TO_PLOT) fig = Figure(size = (800, 600)) @@ -229,10 +231,9 @@ function run_test() if isfile(TEST_IMZML_FILE) # Add spectrum plotting for imzML to match Test Case 1 try - """ # Get the msi data from the imzml println("Plotting a sample spectrum from $TEST_IMZML_FILE...") - msi_data = OpenMSIData(TEST_IMZML_FILE) + msi_data = @time OpenMSIData(TEST_IMZML_FILE) x_coord, y_coord = COORDS_TO_PLOT # Get the x y coordinate spectrum data @@ -275,7 +276,6 @@ function run_test() output_path = joinpath(RESULTS_DIR, "test_imzml_average_spectrum.png") save(output_path, fig) println("SUCCESS: Total spectrum plot saved to $output_path") - """ println("No spectrums tested on this try.") catch e println("ERROR during spectrum plotting in Test Case 3: $e") @@ -284,7 +284,7 @@ function run_test() # Test the plot_slice function try println("Testing plot_slice function on $TEST_IMZML_FILE...") - msi_data = OpenMSIData(TEST_IMZML_FILE) + msi_data = @time OpenMSIData(TEST_IMZML_FILE) @time plot_slice(msi_data, MZ_VALUE_FOR_SLICE, MZ_TOLERANCE, RESULTS_DIR, stage_name="test_imzml_single_slice") # The success message is now inside plot_slice catch e From f35acd1d0feee0eaccd42c1648801726a5e97865 Mon Sep 17 00:00:00 2001 From: Pixelguy14 Date: Thu, 9 Oct 2025 12:00:00 -0600 Subject: [PATCH 05/19] updated functions to iterate faster to create imzml images (find_mass, get_mz_slice, _iterate_spectra_fast, get_total_spectrum) metadata adquisition and saving, updated docstrings, started work for parser to detect 4 cases, profile uncompressed (current), profile compressed, centroid uncompressed, centroid compressed --- app.jl | 78 ++++-- app.jl.html | 35 ++- scripts/build.jl | 23 -- scripts/precompile.jl | 45 ---- src/MSIData.jl | 581 ++++++++++++++++++++++++++++++++---------- src/MSI_src.jl | 2 +- src/imzML.jl | 75 ++++-- 7 files changed, 584 insertions(+), 255 deletions(-) delete mode 100644 scripts/build.jl delete mode 100644 scripts/precompile.jl diff --git a/app.jl b/app.jl index f619549..fbf02b8 100644 --- a/app.jl +++ b/app.jl @@ -18,7 +18,7 @@ using Base.Filesystem: mv # To rename files in the system using Printf # Required for @sprintf macro in colorbar generation # Bring MSIData into App module's scope -using .MSI_src: MSIData, OpenMSIData, GetSpectrum, IterateSpectra, ImzMLSource, _iterate_spectra_fast, MzMLSource, find_mass, ViridisPalette, get_mz_slice, quantize_intensity, save_bitmap, median_filter, save_bitmap, downsample_spectrum, TrIQ +using .MSI_src: MSIData, OpenMSIData, GetSpectrum, IterateSpectra, ImzMLSource, _iterate_spectra_fast, MzMLSource, find_mass, ViridisPalette, get_mz_slice, quantize_intensity, save_bitmap, median_filter, save_bitmap, downsample_spectrum, TrIQ, precompute_analytics include("./julia_imzML_visual.jl") @@ -120,6 +120,13 @@ include("./julia_imzML_visual.jl") # Centralized MSIData object @out msi_data::Union{MSIData, Nothing} = nothing + # Metadata table variables + @in showMetadataDialog = false + @in showMetadataBtn = false + @out metadata_columns = [] + @out metadata_rows = [] + @out btnMetadataDisable = true + # Saves the route where imzML and mzML files are located @out full_route="" @@ -291,31 +298,48 @@ include("./julia_imzML_visual.jl") sTime = time() msi_data = OpenMSIData(full_route) + # --- Pre-compute analytics for performance --- + precompute_analytics(msi_data) + + # --- Prepare metadata for display --- + if msi_data.spectrum_stats_df !== nothing + df = msi_data.spectrum_stats_df + + # Define columns for the key-value summary table + metadata_columns = [ + Dict("name" => "parameter", "label" => "Parameter", "field" => "parameter", "align" => "left"), + Dict("name" => "value", "label" => "Value", "field" => "value", "align" => "left"), + ] + + # Calculate summary statistics + summary_stats = [ + Dict("parameter" => "File Name", "value" => basename(full_route)), + Dict("parameter" => "Number of Spectra", "value" => length(msi_data.spectra_metadata)), + Dict("parameter" => "Image Dimensions", "value" => "$(msi_data.image_dims[1]) x $(msi_data.image_dims[2])"), + Dict("parameter" => "Global Min m/z", "value" => @sprintf("%.4f", msi_data.global_min_mz)), + Dict("parameter" => "Global Max m/z", "value" => @sprintf("%.4f", msi_data.global_max_mz)), + Dict("parameter" => "Mean TIC", "value" => @sprintf("%.2e", mean(df.TIC))), + Dict("parameter" => "Mean BPI", "value" => @sprintf("%.2e", mean(df.BPI))), + Dict("parameter" => "Mean # Points", "value" => @sprintf("%.1f", mean(df.NumPoints))), + ] + + metadata_rows = summary_stats + btnMetadataDisable = false + end + w, h = msi_data.image_dims imgWidth, imgHeight = w > 0 ? (w, h) : (500, 500) fTime = time() eTime = round(fTime - sTime, digits=3) - # msg = "File loaded in $(eTime) seconds. Calculating total spectrum..." - msg = "File loaded in $(eTime) seconds." + msg = "File loaded and indexed in $(eTime) seconds." # Enable UI controls btnStartDisable = !(msi_data.source isa ImzMLSource) btnPlotDisable = false btnSpectraDisable = false SpectraEnabled = true - """ - # --- Automatically generate and display the sum spectrum plot --- - progressSpectraPlot = true - sTime = time() - - plotdata, plotlayout, xSpectraMz, ySpectraMz = sumSpectrumPlot(msi_data) - selectedTab = "tab2" - fTime = time() - eTime = round(fTime - sTime, digits=3) - msg = "Total spectrum plot loaded in $(eTime) seconds." - """ catch e msi_data = nothing msg = "Error loading file: $e" @@ -323,6 +347,7 @@ include("./julia_imzML_visual.jl") btnStartDisable = true btnSpectraDisable = true SpectraEnabled = false + btnMetadataDisable = true @error "File loading failed" exception=(e, catch_backtrace()) finally # This block will always run at the end of the async task @@ -335,6 +360,10 @@ include("./julia_imzML_visual.jl") end end end + + @onbutton showMetadataBtn begin + showMetadataDialog = true + end @onbutton mainProcess @time begin # UI updates immediately @@ -355,7 +384,8 @@ include("./julia_imzML_visual.jl") msg = "Creating image for m/z=$(Nmass) Tol=$(Tol). Please be patient." try # Use the new get_mz_slice with the centralized MSIData object - slice = get_mz_slice(msi_data, Nmass, Tol) + println("get_mz_slice time:") + slice = @time get_mz_slice(msi_data, Nmass, Tol) fig = CairoMakie.Figure(size=(150, 250)) # Container timestamp = string(time_ns()) @@ -364,12 +394,14 @@ include("./julia_imzML_visual.jl") msg = "Incorrect TrIQ values, please adjust accordingly and try again." warning_msg = true else - sliceTriq = TrIQ(slice, colorLevel, triqProb) + println("TrIQ time:") + sliceTriq = @time TrIQ(slice, colorLevel, triqProb) if MFilterEnabled sliceTriq = round.(UInt8, median_filter(sliceTriq)) end sliceTriq = reverse(sliceTriq, dims=2) - save_bitmap(joinpath("public", "TrIQ_$(text_nmass).bmp"), sliceTriq, ViridisPalette) + println("save_bitmap time:") + @time save_bitmap(joinpath("public", "TrIQ_$(text_nmass).bmp"), sliceTriq, ViridisPalette) imgIntT = "/TrIQ_$(text_nmass).bmp?t=$(timestamp)" plotdataImgT, plotlayoutImgT, imgWidth, imgHeight = loadImgPlot(imgIntT) @@ -377,7 +409,8 @@ include("./julia_imzML_visual.jl") msgtriq = "TrIQ image with the Nmass of $(replace(text_nmass, "_" => "."))" colorbar_path = joinpath("public", "colorbar_TrIQ_$(text_nmass).png") - generate_colorbar_image(slice, colorLevel, colorbar_path, use_triq=true, triq_prob=triqProb) + println("generate_colorbar_image time:") + @time generate_colorbar_image(slice, colorLevel, colorbar_path, use_triq=true, triq_prob=triqProb) colorbarT = "/colorbar_TrIQ_$(text_nmass).png?t=$(timestamp)" current_col_triq = "colorbar_TrIQ_$(text_nmass).png" @@ -390,12 +423,14 @@ include("./julia_imzML_visual.jl") selectedTab = "tab1" end else # If we don't use TrIQ - sliceQuant = quantize_intensity(slice, colorLevel) + println("quantize_intensity time:") + sliceQuant = @time quantize_intensity(slice, colorLevel) if MFilterEnabled sliceQuant = round.(UInt8, median_filter(sliceQuant)) end sliceQuant = reverse(sliceQuant, dims=2) - save_bitmap(joinpath("public", "MSI_$(text_nmass).bmp"), sliceQuant, ViridisPalette) + println("save_bitmap time:") + @time save_bitmap(joinpath("public", "MSI_$(text_nmass).bmp"), sliceQuant, ViridisPalette) imgInt = "/MSI_$(text_nmass).bmp?t=$(timestamp)" plotdataImg, plotlayoutImg, imgWidth, imgHeight = loadImgPlot(imgInt) @@ -403,7 +438,8 @@ include("./julia_imzML_visual.jl") msgimg = "Image with the Nmass of $(replace(text_nmass, "_" => "."))" colorbar_path = joinpath("public", "colorbar_MSI_$(text_nmass).png") - generate_colorbar_image(slice, colorLevel, colorbar_path) + println("generate_colorbar_image time:") + @time generate_colorbar_image(slice, colorLevel, colorbar_path) colorbar = "/colorbar_MSI_$(text_nmass).png?t=$(timestamp)" current_col_msi = "colorbar_MSI_$(text_nmass).png" diff --git a/app.jl.html b/app.jl.html index f9c9653..88f0525 100644 --- a/app.jl.html +++ b/app.jl.html @@ -14,9 +14,11 @@
Search for the imzML or mzML file in your system
- -

full route: {{full_route}}

+ + +
@@ -106,6 +108,8 @@ +

{{msg}}

@@ -384,4 +388,29 @@ + + + +
Dataset Summary
+ + + + + + + {{ row.parameter }} + + + {{ row.value }} + + + + + + + + + + + \ No newline at end of file diff --git a/scripts/build.jl b/scripts/build.jl deleted file mode 100644 index 86eafa9..0000000 --- a/scripts/build.jl +++ /dev/null @@ -1,23 +0,0 @@ -# scripts/build.jl -using PackageCompiler -using Pkg - -project_dir = dirname(@__DIR__) -Pkg.activate(project_dir) - -# Get all the direct dependencies from Project.toml -deps = keys(Pkg.project().dependencies) -sysimage_path = joinpath(project_dir, "build", "JuliaMSI_sysimage.so") -precompile_script = joinpath(project_dir, "scripts", "precompile.jl") - -@info "Starting system image compilation. This may take several minutes..." - -create_sysimage( - deps; - sysimage_path=sysimage_path, - precompile_execution_file=precompile_script -) - -@info "Compilation finished!" -@info "You can now run the fast-starting application using the following command:" -@info "julia --project=. -J build/JuliaMSI_sysimage.so start_MSI_GUI.jl" diff --git a/scripts/precompile.jl b/scripts/precompile.jl deleted file mode 100644 index 48b33da..0000000 --- a/scripts/precompile.jl +++ /dev/null @@ -1,45 +0,0 @@ -# scripts/precompile.jl -using Pkg -Pkg.activate(dirname(@__DIR__)) - -using Genie -using HTTP - -@info "Precompiling application..." - -# Load the app. Mmap=false is recommended for compilation -Genie.loadapp(pwd(); Mmap=false) - -@info "Starting server for precompilation." -# Start the server in the background -server = up(1481, "127.0.0.1", async=true) - -# Give the server a moment to start up -sleep(20) - -base_url = "http://127.0.0.1:1481" -@info "Hitting routes to precompile..." - -try - # Make a GET request to the home page - response = HTTP.get(base_url * "/") - @info "GET / -> Status: $(response.status)" - - # === IMPORTANT === - # For best performance, you should add more HTTP requests here - # to hit ALL your important routes and API endpoints. - # This ensures the code for every page gets compiled. - # Example: - # HTTP.get(base_url * "/contact") - # HTTP.post(base_url * "/submit-data", [], "{\"key\":\"value\"}") - -catch e - @error "Could not hit routes during precompilation." exception=(e, catch_backtrace()) - -finally - # Stop the server - @info "Shutting down server." - down() -end - -@info "Precompilation tracing finished." diff --git a/src/MSIData.jl b/src/MSIData.jl index 1bdfb61..aabe63b 100644 --- a/src/MSIData.jl +++ b/src/MSIData.jl @@ -6,7 +6,7 @@ including caching and iteration logic, for handling large mzML and imzML dataset efficiently. """ -using Base64, Libz, Serialization, Printf # For reading binary data +using Base64, Libz, Serialization, Printf, DataFrames, Base.Threads # For reading binary data # Abstract type for different data sources (e.g., mzML, imzML) # This allows dispatching to the correct binary reading logic. @@ -46,8 +46,9 @@ Contains metadata for a single binary data array (m/z or intensity) within a spe - `format`: The data type of the elements (e.g., `Float32`, `Int64`). - `is_compressed`: A boolean flag indicating if the data is compressed (e.g., with zlib). - `offset`: The byte offset of the data within the file (`.ibd` for imzML, `.mzML` for mzML). -- `encoded_length`: The length of the data. For mzML, this is the Base64 encoded length. - For imzML, this is the number of elements in the array. +- `encoded_length`: The length of the data. For uncompressed imzML, this is the number of + elements in the array. For compressed imzML, it is the number of bytes of the compressed + data. For mzML, this is the length of the Base64 encoded string. - `axis_type`: A symbol (`:mz` or `:intensity`) indicating the type of data. """ struct SpectrumAsset @@ -96,9 +97,13 @@ efficient repeated access to spectra. - `spectra_metadata`: A vector of `SpectrumMetadata` for all spectra in the file. - `image_dims`: A tuple `(width, height)` of the spatial dimensions (for imzML). - `coordinate_map`: A matrix mapping `(x, y)` coordinates to a linear spectrum index (for imzML). -- `cache`: A dictionary holding cached spectra. -- `cache_order`: A vector tracking the usage order for the LRU cache. +- `cache`: A dictionary holding cached spectra, mapping index to `(mz, intensity)`. +- `cache_order`: A vector of indices tracking usage for the LRU cache policy. - `cache_size`: The maximum number of spectra to store in the cache. +- `cache_lock`: A `ReentrantLock` to ensure thread-safe access to the cache. +- `global_min_mz`: Cached global minimum m/z value across all spectra. +- `global_max_mz`: Cached global maximum m/z value across all spectra. +- `spectrum_stats_df`: A `DataFrame` containing pre-computed per-spectrum analytics (e.g., TIC, BPI). """ mutable struct MSIData source::MSDataSource @@ -106,13 +111,21 @@ mutable struct MSIData image_dims::Tuple{Int, Int} # (width, height) for imaging data coordinate_map::Union{Matrix{Int}, Nothing} # Maps (x,y) to linear index for imzML - # LRU Cache implementation + # LRU Cache for GetSpectrum cache::Dict{Int, Tuple{Vector, Vector}} cache_order::Vector{Int} # Stores indices, with most recently used at the end cache_size::Int # Max number of spectra in cache + cache_lock::ReentrantLock # To make cache access thread-safe + + # Pre-computed analytics/metadata + global_min_mz::Union{Float64, Nothing} + global_max_mz::Union{Float64, Nothing} + spectrum_stats_df::Union{DataFrame, Nothing} function MSIData(source, metadata, dims, coordinate_map, cache_size) - obj = new(source, metadata, dims, coordinate_map, Dict(), [], cache_size) + obj = new(source, metadata, dims, coordinate_map, + Dict(), [], cache_size, ReentrantLock(), + nothing, nothing, nothing) # Initialize new fields to nothing # Ensure file handles are closed when the object is garbage collected finalizer(obj) do o @@ -132,15 +145,18 @@ end """ read_binary_vector(io::IO, asset::SpectrumAsset) -Reads and decodes a single binary data vector (like m/z or intensity array) +Reads and decodes a single binary data vector (e.g., m/z or intensity array) from a `.mzML` file. The data is expected to be Base64-encoded and may be compressed. This internal function handles: -1. Reading the raw Base64 string. -2. Decoding from Base64. -3. Decompressing the data if `asset.is_compressed` is true. -4. Converting the byte order from network (big-endian) to host order. +1. Reading the raw Base64 string from the file at the specified offset. +2. Decoding the Base64 string into bytes. +3. Decompressing the bytes using zlib if `asset.is_compressed` is true. +4. Interpreting the resulting bytes as a vector of the specified format. + +Note: Byte order conversion (e.g., from little-endian to host) is not performed +by this function and is assumed to be handled by the caller if necessary. # Arguments - `io`: The IO stream of the `.mzML` file. @@ -221,46 +237,75 @@ end """ GetSpectrum(data::MSIData, index::Int) -Retrieves a single spectrum by its index, utilizing a cache for performance. +Retrieves a single spectrum by its index, utilizing a thread-safe LRU cache for performance. +If the spectrum is not in the cache, it is read from disk, and the cache is updated. This function is the core of the "Indexed" and "Cache" access patterns. + +# Arguments +- `data`: The `MSIData` object. +- `index`: The linear index of the spectrum to retrieve. + +# Returns +- A tuple `(mz, intensity)` containing the spectrum's data arrays. """ function GetSpectrum(data::MSIData, index::Int) if index < 1 || index > length(data.spectra_metadata) error("Spectrum index $index out of bounds.") end - # Phase 1: Check the cache - if haskey(data.cache, index) - # Cache Hit: Move item to the end of the LRU list and return from cache - filter!(x -> x != index, data.cache_order) - push!(data.cache_order, index) - return data.cache[index] + # Phase 1: Check the cache (with lock) + lock(data.cache_lock) + try + if haskey(data.cache, index) + # Cache Hit: Move item to the end of the LRU list and return from cache + filter!(x -> x != index, data.cache_order) + push!(data.cache_order, index) + return data.cache[index] + end + finally + unlock(data.cache_lock) end - # Phase 2: Cache Miss - Read from disk + # Phase 2: Cache Miss - Read from disk (no lock) meta = data.spectra_metadata[index] spectrum = read_spectrum_from_disk(data.source, meta) - # Phase 3: Update cache - if data.cache_size > 0 - if length(data.cache) >= data.cache_size - # Evict the least recently used item (at the front of the list) - lru_index = popfirst!(data.cache_order) - delete!(data.cache, lru_index) + # Phase 3: Update cache (with lock) and get final value + return lock(data.cache_lock) do + if haskey(data.cache, index) + # Another thread got here first, use its result + return data.cache[index] end - data.cache[index] = spectrum - push!(data.cache_order, index) - end - return spectrum + # This thread is first to update cache + if data.cache_size > 0 + if length(data.cache) >= data.cache_size + # Evict the least recently used item (at the front of the list) + lru_index = popfirst!(data.cache_order) + delete!(data.cache, lru_index) + end + data.cache[index] = spectrum + push!(data.cache_order, index) + end + return spectrum + end end """ GetSpectrum(data::MSIData, x::Int, y::Int) -Retrieves a single spectrum by its (x, y) coordinates for imaging data. -Utilizes a coordinate map for efficient lookup and then the cache. +Retrieves a single spectrum by its (x, y) coordinates for imaging data (`.imzML`). +This method uses the `coordinate_map` for efficient index lookup and then calls +the indexed `GetSpectrum` method, benefiting from caching. + +# Arguments +- `data`: The `MSIData` object. +- `x`: The x-coordinate of the spectrum. +- `y`: The y-coordinate of the spectrum. + +# Returns +- A tuple `(mz, intensity)` containing the spectrum's data arrays. """ function GetSpectrum(data::MSIData, x::Int, y::Int) if data.coordinate_map === nothing @@ -279,32 +324,152 @@ function GetSpectrum(data::MSIData, x::Int, y::Int) return GetSpectrum(data, index) # Call the existing method end -using Serialization +""" + precompute_analytics(msi_data::MSIData) +Performs a single pass over the entire dataset to pre-compute and cache important +analytics. This function populates the `global_min_mz`, `global_max_mz`, and +`spectrum_stats_df` fields of the `MSIData` object. + +The computed statistics include: +- Global minimum and maximum m/z values. +- Per-spectrum: + - Total Ion Count (TIC) + - Base Peak Intensity (BPI) + - m/z of the base peak + - Number of data points + - Minimum and maximum m/z + +Subsequent calls to functions like `get_total_spectrum` will be much faster +as they can use this cached data. This function modifies the `MSIData` object in-place +and is idempotent. +""" +function precompute_analytics(msi_data::MSIData) + # Idempotency check: If already computed, do nothing. + if msi_data.spectrum_stats_df !== nothing && hasproperty(msi_data.spectrum_stats_df, :MinMZ) + println("Analytics have already been pre-computed.") + return + end + """ + meta = msi_data.spectra_metadata[1] + println("First spectrum:") + println(" mz compressed: $(meta.mz_asset.is_compressed)") + println(" int compressed: $(meta.int_asset.is_compressed)") + println(" mz encoded_length: $(meta.mz_asset.encoded_length)") + println(" int encoded_length: $(meta.int_asset.encoded_length)") + + println("Pre-computing analytics (single pass)...") + """ + start_time = time_ns() + + num_spectra = length(msi_data.spectra_metadata) + + # Initialize variables for global stats + g_min_mz = Inf + g_max_mz = -Inf + + # Initialize vectors for per-spectrum stats + tics = Vector{Float64}(undef, num_spectra) + bpis = Vector{Float64}(undef, num_spectra) + bp_mzs = Vector{Float64}(undef, num_spectra) + num_points = Vector{Int}(undef, num_spectra) + min_mzs = Vector{Float64}(undef, num_spectra) + max_mzs = Vector{Float64}(undef, num_spectra) + + _iterate_spectra_fast(msi_data) do idx, mz, intensity + if isempty(mz) + tics[idx] = 0.0 + bpis[idx] = 0.0 + bp_mzs[idx] = 0.0 + num_points[idx] = 0 + min_mzs[idx] = Inf + max_mzs[idx] = -Inf + return + end + + # Update global m/z range + local_min, local_max = extrema(mz) + g_min_mz = min(g_min_mz, local_min) + g_max_mz = max(g_max_mz, local_max) + min_mzs[idx] = local_min + max_mzs[idx] = local_max + + # Calculate per-spectrum stats + tics[idx] = sum(intensity) + max_int, max_idx = findmax(intensity) + bpis[idx] = max_int + bp_mzs[idx] = mz[max_idx] + num_points[idx] = length(mz) + end + + # Populate the MSIData object + msi_data.global_min_mz = g_min_mz + msi_data.global_max_mz = g_max_mz + msi_data.spectrum_stats_df = DataFrame( + SpectrumID = 1:num_spectra, + TIC = tics, + BPI = bpis, + BasePeakMZ = bp_mzs, + NumPoints = num_points, + MinMZ = min_mzs, + MaxMZ = max_mzs + ) + + duration = (time_ns() - start_time) / 1e9 + @printf "Analytics pre-computation complete in %.2f seconds.\n" duration + + return +end + +""" + get_total_spectrum_imzml(msi_data::MSIData; num_bins::Int=2000) + +Internal function to calculate the total spectrum for an `.imzML` file. + +It uses a fast, two-pass approach: +1. The first pass finds the global m/z range across all spectra. +2. The second pass sums intensities into a pre-defined number of bins. + +This function is highly optimized for `.imzML` by leveraging direct binary +reading and optimized binning logic. It is called by `get_total_spectrum`. +""" function get_total_spectrum_imzml(msi_data::MSIData; num_bins::Int=2000) println("Calculating total spectrum for imzML (2-pass method)...") total_start_time = time_ns() - # 1. First Pass: Find the global m/z range - pass1_start_time = time_ns() - println(" Pass 1: Finding global m/z range...") - global_min_mz = Inf - global_max_mz = -Inf - _iterate_spectra_fast(msi_data) do idx, mz, _ - if !isempty(mz) - local_min, local_max = extrema(mz) - global_min_mz = min(global_min_mz, local_min) - global_max_mz = max(global_max_mz, local_max) + local global_min_mz, global_max_mz + + if msi_data.global_min_mz !== nothing + println(" Using pre-computed m/z range.") + global_min_mz = msi_data.global_min_mz + global_max_mz = msi_data.global_max_mz + else + # 1. First Pass: Find the global m/z range by reading the fast .ibd file + pass1_start_time = time_ns() + println(" Pass 1: Finding global m/z range...") + g_min_mz = Inf + g_max_mz = -Inf + _iterate_spectra_fast(msi_data) do idx, mz, _ + if !isempty(mz) + local_min, local_max = extrema(mz) + g_min_mz = min(g_min_mz, local_min) + g_max_mz = max(g_max_mz, local_max) + end end + pass1_duration = (time_ns() - pass1_start_time) / 1e9 + if !isfinite(g_min_mz) + @warn "Could not determine a valid m/z range for imzML. All spectra might be empty." + return (Float64[], Float64[]) + end + + # Use and cache the result + global_min_mz = g_min_mz + global_max_mz = g_max_mz + msi_data.global_min_mz = global_min_mz + msi_data.global_max_mz = global_max_mz + println(" Global m/z range found and cached: [$(global_min_mz), $(global_max_mz)]") + @printf " (Pass 1 took %.2f seconds)\n" pass1_duration end - pass1_duration = (time_ns() - pass1_start_time) / 1e9 - - if !isfinite(global_min_mz) - @warn "Could not determine a valid m/z range for imzML. All spectra might be empty." - return (Float64[], Float64[]) - end - println(" Global m/z range found: [$(global_min_mz), $(global_max_mz)]") - # 2. Define Bins and precompute constants mz_bins = range(global_min_mz, stop=global_max_mz, length=num_bins) intensity_sum = zeros(Float64, num_bins) @@ -343,86 +508,106 @@ function get_total_spectrum_imzml(msi_data::MSIData; num_bins::Int=2000) pass2_duration = (time_ns() - pass2_start_time) / 1e9 total_duration = (time_ns() - total_start_time) / 1e9 - println("\n--- imzML Profiling ---") - @printf " Pass 1 (I/O only): %.2f seconds\n" pass1_duration + println("\n--- imzML Profiling (Post-Optimization) ---") @printf " Pass 2 (I/O + Binning): %.2f seconds\n" pass2_duration - @printf " Est. Binning Overhead: %.2f seconds\n" (pass2_duration - pass1_duration) @printf " Total Function Time: %.2f seconds\n" total_duration - println("-------------------------\n") + println("----------------------------------------\n") println("Total spectrum calculation complete for imzML.") return (collect(mz_bins), intensity_sum) end +""" + get_total_spectrum_mzml(msi_data::MSIData; num_bins::Int=2000) + +Internal function to calculate the total spectrum for an `.mzML` file. + +It uses a two-pass approach analogous to the `imzML` implementation: +1. The first pass finds the global m/z range by iterating through all spectra. +2. The second pass sums intensities into a pre-defined number of bins. + +This function is called by `get_total_spectrum`. +""" function get_total_spectrum_mzml(msi_data::MSIData; num_bins::Int=2000) - println("Calculating total spectrum for mzML (optimized single-pass method)...") + println("Calculating total spectrum for mzML (2-pass method)...") total_start_time = time_ns() - num_spectra = length(msi_data.spectra_metadata) - if num_spectra == 0 - return (Float64[], Float64[]) - end - temp_path, temp_io = mktemp() - try - # --- Pass 1: Read from source, find m/z range, and write decoded spectra to temp file --- + local global_min_mz, global_max_mz + + if msi_data.global_min_mz !== nothing + println(" Using pre-computed m/z range.") + global_min_mz = msi_data.global_min_mz + global_max_mz = msi_data.global_max_mz + else + # --- Pass 1: Find m/z range and cache it --- pass1_start_time = time_ns() - println(" Pass 1: Caching decoded spectra and finding global m/z range...") - global_min_mz = Inf - global_max_mz = -Inf - + println(" Pass 1: Finding global m/z range...") + g_min_mz = Inf + g_max_mz = -Inf _iterate_spectra_fast(msi_data) do idx, mz, intensity if !isempty(mz) local_min, local_max = extrema(mz) - global_min_mz = min(global_min_mz, local_min) - global_max_mz = max(global_max_mz, local_max) + g_min_mz = min(g_min_mz, local_min) + g_max_mz = max(g_max_mz, local_max) end - Serialization.serialize(temp_io, (mz, intensity)) end - - flush(temp_io) pass1_duration = (time_ns() - pass1_start_time) / 1e9 - if !isfinite(global_min_mz) + if !isfinite(g_min_mz) @warn "Could not determine a valid m/z range for mzML. All spectra might be empty." return (Float64[], Float64[]) end - println(" Global m/z range found: [$(global_min_mz), $(global_max_mz)]") - # --- Pass 2: Read from fast temp file and bin intensities --- - pass2_start_time = time_ns() - println(" Pass 2: Reading from cache and summing intensities into $num_bins bins...") - seekstart(temp_io) - mz_bins = range(global_min_mz, stop=global_max_mz, length=num_bins) - intensity_sum = zeros(Float64, num_bins) - bin_step = step(mz_bins) + # Use and cache the result + global_min_mz = g_min_mz + global_max_mz = g_max_mz + msi_data.global_min_mz = global_min_mz + msi_data.global_max_mz = global_max_mz - while !eof(temp_io) - mz, intensity = Serialization.deserialize(temp_io)::Tuple{AbstractVector, AbstractVector} - if isempty(mz) - continue - end - for i in eachindex(mz) - bin_index = clamp(round(Int, (mz[i] - global_min_mz) / bin_step) + 1, 1, num_bins) + println(" Global m/z range found and cached: [$(global_min_mz), $(global_max_mz)]") + @printf " (Pass 1 took %.2f seconds)\n" pass1_duration + end + + # --- Pass 2: Bin intensities --- + pass2_start_time = time_ns() + println(" Pass 2: Summing intensities into $num_bins bins...") + + mz_bins = range(global_min_mz, stop=global_max_mz, length=num_bins) + intensity_sum = zeros(Float64, num_bins) + bin_step = step(mz_bins) + inv_bin_step = 1.0 / bin_step # Precompute reciprocal + min_mz = global_min_mz + + _iterate_spectra_fast(msi_data) do idx, mz, intensity + if isempty(mz) + return + end + + @inbounds for i in eachindex(mz) + # Calculate raw bin index + raw_index = (mz[i] - min_mz) * inv_bin_step + 1.0 + bin_index = trunc(Int, raw_index) + + # Manual bounds checking + if 1 <= bin_index <= num_bins intensity_sum[bin_index] += intensity[i] + elseif bin_index < 1 + intensity_sum[1] += intensity[i] + else # bin_index > num_bins + intensity_sum[num_bins] += intensity[i] end end - pass2_duration = (time_ns() - pass2_start_time) / 1e9 - - total_duration = (time_ns() - total_start_time) / 1e9 - println("\n--- mzML Profiling ---") - @printf " Pass 1 (Read+Decode+Cache): %.2f seconds\n" pass1_duration - @printf " Pass 2 (Read Cache+Bin): %.2f seconds\n" pass2_duration - @printf " Total Function Time: %.2f seconds\n" total_duration - println("----------------------\n") - - println("Total spectrum calculation complete for mzML.") - return (collect(mz_bins), intensity_sum) - - finally - close(temp_io) - rm(temp_path, force=true) - println(" Temporary cache file removed.") end + pass2_duration = (time_ns() - pass2_start_time) / 1e9 + + total_duration = (time_ns() - total_start_time) / 1e9 + println("\n--- mzML Profiling (Post-Optimization) ---") + @printf " Pass 2 (I/O + Binning): %.2f seconds\n" pass2_duration + @printf " Total Function Time: %.2f seconds\n" total_duration + println("----------------------------------------\n") + + println("Total spectrum calculation complete for mzML.") + return (collect(mz_bins), intensity_sum) end """ @@ -443,10 +628,10 @@ function get_total_spectrum(msi_data::MSIData; num_bins::Int=2000) end """ - get_average_spectrum(msi_data::MSIData; num_bins::Int=20000) -> Tuple{Vector{Float64}, Vector{Float64}} + get_average_spectrum(msi_data::MSIData; num_bins::Int=2000) -> Tuple{Vector{Float64}, Vector{Float64}} Calculates the average of all spectra in the dataset by binning. -This is effectively the total ion chromatogram (TIC) divided by the number of spectra. +This is effectively the total spectrum divided by the number of spectra. Returns a tuple containing two vectors: the binned m/z axis and the averaged intensities. """ @@ -481,8 +666,9 @@ end """ IterateSpectra(data::MSIData) -Returns an iterator that yields each spectrum, processing the file sequentially -with minimal memory overhead. This iterator supports caching via `GetSpectrum`. +Returns an iterator that yields each spectrum, processing the file sequentially. +This iterator is useful for processing all spectra in a loop and benefits from +the caching implemented in `GetSpectrum`. This function is the core of the "Event-driven" access pattern. """ @@ -524,49 +710,168 @@ end # --- High-performance Internal Iterator --- # """ - _iterate_spectra_fast_impl(f::Function, data::MSIData, source::ImzMLSource) + read_compressed_array(io::IO, asset::SpectrumAsset, format::Type) -Internal implementation of the fast iterator for `.imzML` files. It reads -data directly from the `.ibd` file stream and reuses pre-allocated buffers -to minimize memory allocations and overhead, making it ideal for bulk processing tasks. +Reads a single data array (m/z or intensity) from an `.ibd` file stream, +handling both compressed and uncompressed data. + +- If `asset.is_compressed` is true, it reads the compressed bytes, inflates + them using zlib, and reinterprets the result as a vector of the given `format`. +- If false, it reads the uncompressed data directly into a vector. + +# Arguments +- `io`: The IO stream of the `.ibd` file. +- `asset`: The `SpectrumAsset` for the array. +- `format`: The data type of the elements in the array. + +# Returns +- A `Vector` containing the data. """ -function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::ImzMLSource) - # Optimized implementation: allocates arrays per spectrum and uses an efficient I/O pattern. +function read_compressed_array(io::IO, asset::SpectrumAsset, format::Type) + seek(io, asset.offset) + + if asset.is_compressed + # Read compressed bytes + compressed_bytes = read(io, asset.encoded_length) + + local decompressed_bytes + try + decompressed_bytes = Libz.inflate(compressed_bytes) + catch e + @error "ZLIB DECOMPRESSION FAILED. This is likely due to an incorrect offset or corrupt data in the .ibd file." + @error "Asset offset: $(asset.offset), Encoded length: $(asset.encoded_length)" + # Print first 16 bytes to stderr for diagnosis + bytes_to_print = min(16, length(compressed_bytes)) + @error "First $bytes_to_print bytes of the data chunk we tried to decompress:" + println(stderr, view(compressed_bytes, 1:bytes_to_print)) + rethrow(e) + end + + # Use an IOBuffer to safely read the data, avoiding reinterpret errors + # if the decompressed size is not a perfect multiple of the element size. + bytes_io = IOBuffer(decompressed_bytes) + n_elements = bytes_io.size ÷ sizeof(format) + array = Array{format}(undef, n_elements) + read!(bytes_io, array) + return array + else + # Read uncompressed data directly + # For uncompressed imzML, encoded_length is the number of elements + array = Vector{format}(undef, asset.encoded_length) + read!(io, array) + return array + end +end + +""" + _iterate_uncompressed_fast(f::Function, data::MSIData, source::ImzMLSource) + +A highly optimized iterator for uncompressed `.imzML` data. + +It pre-allocates large buffers for m/z and intensity arrays and reuses them +for each spectrum by creating views. This minimizes memory allocations and +is significantly faster for bulk processing than reading spectra one by one. + +# Arguments +- `f`: A function to execute for each spectrum, with the signature `f(index, mz_view, int_view)`. +- `data`: The `MSIData` object. +- `source`: The `ImzMLSource`. +""" +function _iterate_uncompressed_fast(f::Function, data::MSIData, source::ImzMLSource) + # Optimized path for uncompressed data using buffer reuse + max_points = maximum(meta -> meta.mz_asset.encoded_length, data.spectra_metadata) + mz_buffer = Vector{source.mz_format}(undef, max_points) + int_buffer = Vector{source.intensity_format}(undef, max_points) + for i in 1:length(data.spectra_metadata) meta = data.spectra_metadata[i] nPoints = meta.mz_asset.encoded_length if nPoints == 0 + f(i, view(mz_buffer, 0:-1), view(int_buffer, 0:-1)) + continue + end + + mz_view = view(mz_buffer, 1:nPoints) + int_view = view(int_buffer, 1:nPoints) + + if meta.mz_asset.offset < meta.int_asset.offset + seek(source.ibd_handle, meta.mz_asset.offset) + read!(source.ibd_handle, mz_view) + read!(source.ibd_handle, int_view) + else + seek(source.ibd_handle, meta.int_asset.offset) + read!(source.ibd_handle, int_view) + read!(source.ibd_handle, mz_view) + end + + mz_view .= ltoh.(mz_view) + int_view .= ltoh.(int_view) + f(i, mz_view, int_view) + end +end + +""" + _iterate_compressed_fast(f::Function, data::MSIData, source::ImzMLSource) + +An iterator for `.imzML` datasets that contain compressed spectra. + +This function iterates through each spectrum and reads its data individually, +decompressing it on the fly if necessary. Because the size of decompressed +data is not known in advance, this path cannot use the buffer-reuse optimization +and will be slower and allocate more memory than `_iterate_uncompressed_fast`. + +# Arguments +- `f`: A function to execute for each spectrum, with the signature `f(index, mz_array, int_array)`. +- `data`: The `MSIData` object. +- `source`: The `ImzMLSource`. +""" +function _iterate_compressed_fast(f::Function, data::MSIData, source::ImzMLSource) + # Path for datasets containing at least one compressed spectrum. + # This path reads and decompresses each spectrum individually. + for i in 1:length(data.spectra_metadata) + meta = data.spectra_metadata[i] + + if meta.mz_asset.encoded_length == 0 && meta.int_asset.encoded_length == 0 f(i, source.mz_format[], source.intensity_format[]) continue end - # Allocate fresh arrays for each spectrum. - mz = Vector{source.mz_format}(undef, nPoints) - intensity = Vector{source.intensity_format}(undef, nPoints) + # Read and decompress each array + mz_array = read_compressed_array(source.ibd_handle, meta.mz_asset, source.mz_format) + intensity_array = read_compressed_array(source.ibd_handle, meta.int_asset, source.intensity_format) + + mz_array .= ltoh.(mz_array) + intensity_array .= ltoh.(intensity_array) + + f(i, mz_array, intensity_array) + end +end - # --- I/O OPTIMIZATION: Seek only once per spectrum --- - # The mz and intensity data are contiguous, so after reading the first, - # we can immediately read the second without a costly second seek. - if meta.mz_asset.offset < meta.int_asset.offset - # m/z is first, seek to it. - seek(source.ibd_handle, meta.mz_asset.offset) - # Read m/z, then immediately read intensity. - read!(source.ibd_handle, mz) - read!(source.ibd_handle, intensity) - else - # intensity is first, seek to it. - seek(source.ibd_handle, meta.int_asset.offset) - # Read intensity, then immediately read m/z. - read!(source.ibd_handle, intensity) - read!(source.ibd_handle, mz) - end +""" + _iterate_spectra_fast_impl(f::Function, data::MSIData, source::ImzMLSource) - # Convert byte order. - mz .= ltoh.(mz) - intensity .= ltoh.(intensity) +Internal implementation of the fast iterator for `.imzML` files. It reads +data directly from the `.ibd` file stream. - f(i, mz, intensity) +This function acts as a dispatcher: +- If any spectrum is compressed, it uses a slower path that decompresses each spectrum individually. +- If all spectra are uncompressed, it uses a highly optimized path that reuses pre-allocated + buffers to minimize memory allocations and overhead. +""" +function _iterate_spectra_fast_impl(f::Function, data::MSIData, source::ImzMLSource) + if isempty(data.spectra_metadata) + return + end + + # Check if ANY spectra are compressed and dispatch to the appropriate implementation + any_compressed = any(meta -> meta.mz_asset.is_compressed || meta.int_asset.is_compressed, + data.spectra_metadata) + + if any_compressed + _iterate_compressed_fast(f, data, source) + else + _iterate_uncompressed_fast(f, data, source) end end @@ -606,4 +911,4 @@ It dispatches to a specialized implementation based on the data source type function _iterate_spectra_fast(f::Function, data::MSIData) # Dispatch to the correct implementation based on the source type _iterate_spectra_fast_impl(f, data, data.source) -end +end \ No newline at end of file diff --git a/src/MSI_src.jl b/src/MSI_src.jl index 59a40c4..444dce5 100644 --- a/src/MSI_src.jl +++ b/src/MSI_src.jl @@ -1,7 +1,7 @@ module MSI_src # Export the public API -export OpenMSIData, GetSpectrum, IterateSpectra, ImportMzmlFile, load_slices, plot_slices, plot_slice, get_total_spectrum, get_average_spectrum, LoadMzml, LoadSpectra +export OpenMSIData, GetSpectrum, IterateSpectra, ImportMzmlFile, load_slices, plot_slices, plot_slice, get_total_spectrum, get_average_spectrum, LoadMzml, LoadSpectra, precompute_analytics # Include all source files directly into the main module include("MSIData.jl") diff --git a/src/imzML.jl b/src/imzML.jl index 61f60c3..36b4dd0 100644 --- a/src/imzML.jl +++ b/src/imzML.jl @@ -268,8 +268,7 @@ end find_mass(mz_array, intensity_array, target_mass, tolerance) Finds the intensity of the most intense peak within a mass tolerance window. -This modernized version is more robust than a simple binary search as it -correctly handles multiple peaks within the tolerance window. +This optimized version uses binary search for efficiency. # Returns - The intensity (`Float64`) of the peak if found, otherwise `0.0`. @@ -278,20 +277,22 @@ function find_mass(mz_array, intensity_array, target_mass, tolerance) lower_bound = target_mass - tolerance upper_bound = target_mass + tolerance - max_intensity = 0.0 - found = false + # Use binary search to find the start and end of the m/z window + start_idx = searchsortedfirst(mz_array, lower_bound) + end_idx = searchsortedlast(mz_array, upper_bound) - # Iterate through the spectrum to find the highest intensity peak in the window - for i in eachindex(mz_array) - if lower_bound <= mz_array[i] <= upper_bound - if intensity_array[i] > max_intensity - max_intensity = intensity_array[i] - found = true - end - end + # If the window is empty, return 0.0 + if start_idx > end_idx + return 0.0 end - return found ? max_intensity : 0.0 + # Find the maximum intensity within the identified window, optimized with @inbounds and @simd + max_intensity = intensity_array[start_idx] + @inbounds @simd for i in (start_idx + 1):end_idx + max_intensity = max(max_intensity, intensity_array[i]) + end + + return max_intensity end """ @@ -362,22 +363,48 @@ This is a performant function that iterates through spectra once. """ function get_mz_slice(data::MSIData, mass::Real, tolerance::Real) width, height = data.image_dims - slice_matrix = zeros(Float64, height, width) # Note: height, width for (y,x) indexing + slice_matrix = zeros(Float64, height, width) - _iterate_spectra_fast(data) do spec_idx, mz_array, intensity_array - meta = data.spectra_metadata[spec_idx] - - intensity = find_mass(mz_array, intensity_array, mass, tolerance) - - if intensity > 0.0 - # Populate the matrix using (y, x) indexing - if 1 <= meta.x <= width && 1 <= meta.y <= height - slice_matrix[meta.y, meta.x] = intensity + # INTELLIGENT LOADING: Ensure analytics are computed for filtering. + if data.spectrum_stats_df === nothing || !hasproperty(data.spectrum_stats_df, :MinMZ) + println("Per-spectrum metadata not found. Running one-time analytics computation...") + precompute_analytics(data) + end + + println("Using high-performance sequential iterator...") + target_min = mass - tolerance + target_max = mass + tolerance + stats_df = data.spectrum_stats_df + + # 1. Find all candidate spectra first for efficient filtering + candidate_indices = Set{Int}() + for i in 1:length(data.spectra_metadata) + spec_min_mz = stats_df.MinMZ[i] + spec_max_mz = stats_df.MaxMZ[i] + if target_max >= spec_min_mz && target_min <= spec_max_mz + push!(candidate_indices, i) + end + end + + println("Found $(length(candidate_indices)) candidate spectra (filtered from $(length(data.spectra_metadata)))") + + # 2. Iterate using the optimized, low-allocation iterator + results_count = 0 + _iterate_spectra_fast(data) do idx, mz_array, intensity_array + # Process only the spectra that are candidates + if idx in candidate_indices + intensity = find_mass(mz_array, intensity_array, mass, tolerance) + if intensity > 0.0 + meta = data.spectra_metadata[idx] + if 1 <= meta.x <= width && 1 <= meta.y <= height + slice_matrix[meta.y, meta.x] = intensity + results_count += 1 + end end end end - # The original function had a NaN replacement, which is good practice to keep. + println("Populated $results_count pixels with intensity data") replace!(slice_matrix, NaN => 0.0) return slice_matrix end From 2fd2b905139ae896e6009c791bfe6e4655a9f1cd Mon Sep 17 00:00:00 2001 From: Pixelguy14 Date: Mon, 13 Oct 2025 22:05:56 -0600 Subject: [PATCH 06/19] altered imzML parser to be compatible with timsconverter, small improvements on the UI, other small bug fixes done --- README.md | 2 +- app.jl | 245 +++++++++++++----- app.jl.html | 109 ++++---- julia_imzML_visual.jl | 98 +++++++- public/css/genieapp.css | 10 +- src/MSIData.jl | 72 +++++- src/ParserHelpers.jl | 22 +- src/imzML.jl | 533 ++++++++++++++++++++++++++++++++++------ test/run_tests.jl | 65 ++++- 9 files changed, 916 insertions(+), 240 deletions(-) diff --git a/README.md b/README.md index 2affa30..489bd0f 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ https://codeberg.org/LabABI/JuliaMSI ~/Downloads/JuliaMSI-main/juliamsi 2. Without entering the Julia environment, launch the project in your terminal with the following command (which works for all operating systems): ``` - julia --project=. start_MSI_GUI.jl + julia --threads auto --project=. start_MSI_GUI.jl ``` 3. After the script has finished loading, you can open a [page](http://127.0.0.1:1481/) in your browser with the web app running. diff --git a/app.jl b/app.jl index fbf02b8..e1f5100 100644 --- a/app.jl +++ b/app.jl @@ -159,6 +159,14 @@ include("./julia_imzML_visual.jl") ## Plots # Local image to plot layoutImg=PlotlyBase.Layout( + title=PlotlyBase.attr( + text="", + font=PlotlyBase.attr( + family="Roboto, Lato, sans-serif", + size=14, + color="black" + ) + ), xaxis=PlotlyBase.attr( visible=false, scaleanchor="y", @@ -185,7 +193,14 @@ include("./julia_imzML_visual.jl") @out plotlayoutImgTComp=layoutImg # Interface Plot Spectrum layoutSpectra=PlotlyBase.Layout( - title="Spectrum plot", + title=PlotlyBase.attr( + text="Spectrum plot", + font=PlotlyBase.attr( + family="Roboto, Lato, sans-serif", + size=18, + color="black" + ) + ), hovermode="closest", xaxis=PlotlyBase.attr( title="m/z", @@ -215,7 +230,14 @@ include("./julia_imzML_visual.jl") # Interface Plot Surface layoutContour=PlotlyBase.Layout( - title="2D Topographic map", + title=PlotlyBase.attr( + text="2D Topographic map", + font=PlotlyBase.attr( + family="Roboto, Lato, sans-serif", + size=18, + color="black" + ) + ), xaxis=PlotlyBase.attr( visible=false, scaleanchor="y" @@ -226,7 +248,7 @@ include("./julia_imzML_visual.jl") margin=attr(l=0,r=0,t=100,b=0,pad=0) ) # Dummy 2D surface plot - traceContour=PlotlyBase.scatter(x=Vector{Float64}(), y=Vector{Float64}(), mode="lines") + traceContour=PlotlyBase.contour(x=Vector{Float64}(), y=Vector{Float64}(), mode="lines") # Create conection to frontend @out plotdataC=[traceContour] @out plotlayoutC=layoutContour @@ -234,7 +256,14 @@ include("./julia_imzML_visual.jl") # Interface Plot 3d # Define the layout for the 3D plot layout3D=PlotlyBase.Layout( - title="3D Surface plot", + title=PlotlyBase.attr( + text="3D Surface plot", + font=PlotlyBase.attr( + family="Roboto, Lato, sans-serif", + size=18, + color="black" + ) + ), scene=attr( xaxis_title="X", yaxis_title="Y", @@ -386,69 +415,107 @@ include("./julia_imzML_visual.jl") # Use the new get_mz_slice with the centralized MSIData object println("get_mz_slice time:") slice = @time get_mz_slice(msi_data, Nmass, Tol) - fig = CairoMakie.Figure(size=(150, 250)) # Container - timestamp = string(time_ns()) - - if triqEnabled # If we have TrIQ - if triqProb < 0.8 || triqProb > 1 - msg = "Incorrect TrIQ values, please adjust accordingly and try again." - warning_msg = true - else - println("TrIQ time:") - sliceTriq = @time TrIQ(slice, colorLevel, triqProb) - if MFilterEnabled - sliceTriq = round.(UInt8, median_filter(sliceTriq)) - end - sliceTriq = reverse(sliceTriq, dims=2) - println("save_bitmap time:") - @time save_bitmap(joinpath("public", "TrIQ_$(text_nmass).bmp"), sliceTriq, ViridisPalette) + # Failsafe check for empty slice + if all(iszero, slice) + msg = "No intensity data found for m/z = $Nmass with tolerance = $Tol. The resulting image is black. Please consider using a larger tolerance." + warning_msg = true + + # Generate a black image but skip the colorbar that would crash + timestamp = string(time_ns()) + text_nmass = replace(string(Nmass), "." => "_") + sliceQuant = zeros(UInt8, size(slice)) + + if triqEnabled + @time save_bitmap(joinpath("public", "TrIQ_$(text_nmass).bmp"), sliceQuant, ViridisPalette) imgIntT = "/TrIQ_$(text_nmass).bmp?t=$(timestamp)" plotdataImgT, plotlayoutImgT, imgWidth, imgHeight = loadImgPlot(imgIntT) current_triq = "TrIQ_$(text_nmass).bmp" - msgtriq = "TrIQ image with the Nmass of $(replace(text_nmass, "_" => "."))" - - colorbar_path = joinpath("public", "colorbar_TrIQ_$(text_nmass).png") - println("generate_colorbar_image time:") - @time generate_colorbar_image(slice, colorLevel, colorbar_path, use_triq=true, triq_prob=triqProb) - colorbarT = "/colorbar_TrIQ_$(text_nmass).png?t=$(timestamp)" - current_col_triq = "colorbar_TrIQ_$(text_nmass).png" - + msgtriq = "TrIQ image with the Nmass of $(replace(text_nmass, "_" => ".")) (No data)" + colorbarT = "" # Clear colorbar + current_col_triq = "" triq_bmp = sort(filter(filename -> startswith(filename, "TrIQ_") && endswith(filename, ".bmp"), readdir("public")), lt=natural) - col_triq_png = sort(filter(filename -> startswith(filename, "colorbar_TrIQ_") && endswith(filename, ".png"), readdir("public")), lt=natural) - - fTime = time() - eTime = round(fTime - sTime, digits=3) - msg = "The TrIQ image has been created in $(eTime) seconds successfully inside the 'public' folder of the app" selectedTab = "tab1" + else + @time save_bitmap(joinpath("public", "MSI_$(text_nmass).bmp"), sliceQuant, ViridisPalette) + imgInt = "/MSI_$(text_nmass).bmp?t=$(timestamp)" + plotdataImg, plotlayoutImg, imgWidth, imgHeight = loadImgPlot(imgInt) + current_msi = "MSI_$(text_nmass).bmp" + msgimg = "Image with the Nmass of $(replace(text_nmass, "_" => ".")) (No data)" + colorbar = "" # Clear colorbar + current_col_msi = "" + msi_bmp = sort(filter(filename -> startswith(filename, "MSI_") && endswith(filename, ".bmp"), readdir("public")), lt=natural) + selectedTab = "tab0" end - else # If we don't use TrIQ - println("quantize_intensity time:") - sliceQuant = @time quantize_intensity(slice, colorLevel) - if MFilterEnabled - sliceQuant = round.(UInt8, median_filter(sliceQuant)) - end - sliceQuant = reverse(sliceQuant, dims=2) - println("save_bitmap time:") - @time save_bitmap(joinpath("public", "MSI_$(text_nmass).bmp"), sliceQuant, ViridisPalette) - - imgInt = "/MSI_$(text_nmass).bmp?t=$(timestamp)" - plotdataImg, plotlayoutImg, imgWidth, imgHeight = loadImgPlot(imgInt) - current_msi = "MSI_$(text_nmass).bmp" - msgimg = "Image with the Nmass of $(replace(text_nmass, "_" => "."))" - - colorbar_path = joinpath("public", "colorbar_MSI_$(text_nmass).png") - println("generate_colorbar_image time:") - @time generate_colorbar_image(slice, colorLevel, colorbar_path) - colorbar = "/colorbar_MSI_$(text_nmass).png?t=$(timestamp)" - current_col_msi = "colorbar_MSI_$(text_nmass).png" - - msi_bmp = sort(filter(filename -> startswith(filename, "MSI_") && endswith(filename, ".bmp"), readdir("public")), lt=natural) - col_msi_png = sort(filter(filename -> startswith(filename, "colorbar_MSI_") && endswith(filename, ".png"), readdir("public")), lt=natural) - selectedTab = "tab0" fTime = time() eTime = round(fTime - sTime, digits=3) - msg = "The image has been created in $(eTime) seconds successfully inside the 'public' folder of the app" + # The warning message is already set above + else + # Original path for when data is found + fig = CairoMakie.Figure(size=(150, 250)) # Container + timestamp = string(time_ns()) + + if triqEnabled # If we have TrIQ + if triqProb < 0.8 || triqProb > 1 + msg = "Incorrect TrIQ values, please adjust accordingly and try again." + warning_msg = true + else + println("TrIQ time:") + sliceTriq = @time TrIQ(slice, colorLevel, triqProb) + if MFilterEnabled + sliceTriq = round.(UInt8, median_filter(sliceTriq)) + end + + println("save_bitmap time:") + @time save_bitmap(joinpath("public", "TrIQ_$(text_nmass).bmp"), sliceTriq, ViridisPalette) + + imgIntT = "/TrIQ_$(text_nmass).bmp?t=$(timestamp)" + plotdataImgT, plotlayoutImgT, imgWidth, imgHeight = loadImgPlot(imgIntT) + current_triq = "TrIQ_$(text_nmass).bmp" + msgtriq = "TrIQ image with the Nmass of $(replace(text_nmass, "_" => "."))" + + colorbar_path = joinpath("public", "colorbar_TrIQ_$(text_nmass).png") + println("generate_colorbar_image time:") + @time generate_colorbar_image(slice, colorLevel, colorbar_path, use_triq=true, triq_prob=triqProb) + colorbarT = "/colorbar_TrIQ_$(text_nmass).png?t=$(timestamp)" + current_col_triq = "colorbar_TrIQ_$(text_nmass).png" + + triq_bmp = sort(filter(filename -> startswith(filename, "TrIQ_") && endswith(filename, ".bmp"), readdir("public")), lt=natural) + col_triq_png = sort(filter(filename -> startswith(filename, "colorbar_TrIQ_") && endswith(filename, ".png"), readdir("public")), lt=natural) + + fTime = time() + eTime = round(fTime - sTime, digits=3) + msg = "The TrIQ image has been created in $(eTime) seconds successfully inside the 'public' folder of the app" + selectedTab = "tab1" + end + else # If we don't use TrIQ + println("quantize_intensity time:") + sliceQuant = @time quantize_intensity(slice, colorLevel) + if MFilterEnabled + sliceQuant = round.(UInt8, median_filter(sliceQuant)) + end + + println("save_bitmap time:") + @time save_bitmap(joinpath("public", "MSI_$(text_nmass).bmp"), sliceQuant, ViridisPalette) + + imgInt = "/MSI_$(text_nmass).bmp?t=$(timestamp)" + plotdataImg, plotlayoutImg, imgWidth, imgHeight = loadImgPlot(imgInt) + current_msi = "MSI_$(text_nmass).bmp" + msgimg = "Image with the Nmass of $(replace(text_nmass, "_" => "."))" + + colorbar_path = joinpath("public", "colorbar_MSI_$(text_nmass).png") + println("generate_colorbar_image time:") + @time generate_colorbar_image(slice, colorLevel, colorbar_path) + colorbar = "/colorbar_MSI_$(text_nmass).png?t=$(timestamp)" + current_col_msi = "colorbar_MSI_$(text_nmass).png" + + msi_bmp = sort(filter(filename -> startswith(filename, "MSI_") && endswith(filename, ".bmp"), readdir("public")), lt=natural) + col_msi_png = sort(filter(filename -> startswith(filename, "colorbar_MSI_") && endswith(filename, ".png"), readdir("public")), lt=natural) + selectedTab = "tab0" + fTime = time() + eTime = round(fTime - sTime, digits=3) + msg = "The image has been created in $(eTime) seconds successfully inside the 'public' folder of the app" + end end catch e msg = "There was an error creating the image: $e" @@ -624,6 +691,7 @@ include("./julia_imzML_visual.jl") else traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImg=[traceImg] + plotlayoutImg=layoutImg msgimg="" end end @@ -651,6 +719,7 @@ include("./julia_imzML_visual.jl") else traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImg=[traceImg] + plotlayoutImg=layoutImg msgimg="" end end @@ -679,6 +748,7 @@ include("./julia_imzML_visual.jl") else traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgT=[traceImg] + plotlayoutImgT=layoutImg msgtriq="" end end @@ -706,6 +776,7 @@ include("./julia_imzML_visual.jl") else traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgT=[traceImg] + plotlayoutImgT=layoutImg msgtriq="" end end @@ -735,6 +806,7 @@ include("./julia_imzML_visual.jl") else traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgComp=[traceImg] + plotlayoutImgComp=layoutImg msgimgComp="" end end @@ -763,6 +835,7 @@ include("./julia_imzML_visual.jl") else traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgComp=[traceImg] + plotlayoutImgComp=layoutImg msgimgComp="" end end @@ -791,6 +864,7 @@ include("./julia_imzML_visual.jl") else traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgTComp=[traceImg] + plotlayoutImgTComp=layoutImg msgtriqComp="" end end @@ -819,6 +893,7 @@ include("./julia_imzML_visual.jl") else traceImg=PlotlyBase.heatmap(x=Vector{Float64}(), y=Vector{Float64}()) plotdataImgTComp=[traceImg] + plotlayoutImgTComp=layoutImg msgtriqComp="" end end @@ -1026,9 +1101,20 @@ include("./julia_imzML_visual.jl") @onchange data_click begin if selectedTab == "tab1" || selectedTab == "tab0" # This is for the image heatmaps - cursor_data = data_click["cursor"] - x = Int32(round(cursor_data["x"])) - y = Int32(round(cursor_data["y"])) # y is negative in the UI + cursor_data = get(data_click, "cursor", nothing) + if cursor_data === nothing + return + end + + x_val = get(cursor_data, "x", nothing) + y_val = get(cursor_data, "y", nothing) + + if x_val === nothing || y_val === nothing + return # Do nothing if coordinates are not provided by the event + end + + x = Int32(round(x_val)) + y = Int32(round(y_val)) # y is negative in the UI # Update the reactive coordinates, which will trigger the crosshair update xCoord = clamp(x, 1, imgWidth) @@ -1107,3 +1193,40 @@ end # Register a new route and the page that will be loaded on access @page("/", "app.jl.html") end + +#= +function __init__() + @async begin + println("Pre-compiling functions at startup...") + + # Create a dummy MSIData object to be used for pre-compilation + dummy_source = ImzMLSource("dummy.ibd", Float32, Float32) + dummy_meta = MSI_src.SpectrumMetadata(0,0,"",MSI_src.UNKNOWN, MSI_src.SpectrumAsset(Float32,false,0,0,:mz), MSI_src.SpectrumAsset(Float32,false,0,0,:intensity)) + dummy_msi_data = MSIData(dummy_source, [dummy_meta], (1,1), zeros(Int,1,1), 0) + + # Pre-compile functions from btnSearch + try OpenMSIData("dummy.imzML") catch end + try precompute_analytics(dummy_msi_data) catch end + + # Pre-compile functions from mainProcess + try get_mz_slice(dummy_msi_data, 1.0, 1.0) catch end + try TrIQ(zeros(10,10), 256, 0.98) catch end + try quantize_intensity(zeros(10,10), 256) catch end + + dummy_bmp_path = joinpath("public", "dummy.bmp") + dummy_png_path = joinpath("public", "dummy.png") + try + save_bitmap(dummy_bmp_path, zeros(UInt8, 10, 10), ViridisPalette) + loadImgPlot("/dummy.bmp") + generate_colorbar_image(zeros(10,10), 256, dummy_png_path) + catch e + @warn "Pre-compilation step failed (this is expected if dummy files can't be created/read)" + finally + rm(dummy_bmp_path, force=true) + rm(dummy_png_path, force=true) + end + + println("Pre-compilation finished.") + end +end +=# diff --git a/app.jl.html b/app.jl.html index 88f0525..72693eb 100644 --- a/app.jl.html +++ b/app.jl.html @@ -1,5 +1,4 @@ -