Julian/JuliaMSI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improved preprocessing UI, added selective spectrum plot accordig to type (centroid, profile), potentially fixed a bug to retrieve folders on dropdown selectors in startup, improved preprocessing functions and test suite, testing a version control for preprocessing steps

Browse Source
This commit is contained in:
Pixelguy14 2025-11-17 13:31:37 -06:00
parent edfbb77f58
commit 8fa92dc0e5
15 changed files with 1758 additions and 1016 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
Manifest.toml
View File

@ -2,7 +2,12 @@
julia_version = "1.11.7"
manifest_format = "2.0"
project_hash = "38f13d997585a9af185f1863ff626d584a7e791e"
project_hash = "7843f141174fe17a3abc4f25c3819d3195459b5a"
[[deps.ANSIColoredPrinters]]
git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c"
uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9"
version = "0.0.1"
[[deps.ATK_jll]]
deps = ["Artifacts", "Glib_jll", "JLLWrappers", "Libdl"]
@ -159,6 +164,11 @@ git-tree-sha1 = "bca794632b8a9bbe159d56bf9e31c422671b35e0"
uuid = "18cc8868-cbac-4acf-b575-c8ff214dc66f"
version = "1.3.2"
[[deps.Bessels]]
git-tree-sha1 = "4435559dc39793d53a9e3d278e185e920b4619ef"
uuid = "0e736298-9ec6-45e8-9647-e4fc86a2fe38"
version = "0.2.8"
[[deps.BitFlags]]
git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d"
uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35"
@ -289,6 +299,12 @@ git-tree-sha1 = "980f01d6d3283b3dbdfd7ed89405f96b7256ad57"
uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
version = "2.0.1"
[[deps.CodecBase]]
deps = ["TranscodingStreams"]
git-tree-sha1 = "40956acdbef3d8c7cc38cba42b56034af8f8581a"
uuid = "6c391c72-fb7b-5838-ba82-7cfb1bcfecbf"
version = "0.3.4"
[[deps.CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"]
git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9"
@ -397,6 +413,12 @@ weakdeps = ["IntervalSets", "LinearAlgebra", "StaticArrays"]
ConstructionBaseLinearAlgebraExt = "LinearAlgebra"
ConstructionBaseStaticArraysExt = "StaticArrays"
[[deps.ContinuousWavelets]]
deps = ["AbstractFFTs", "Documenter", "FFTW", "Interpolations", "LinearAlgebra", "SpecialFunctions", "Wavelets"]
git-tree-sha1 = "6e883d34f7040ab2ec36c60c738ae53864819508"
uuid = "96eb917e-2868-4417-9cb6-27e7ff17528f"
version = "1.1.7"
[[deps.Contour]]
git-tree-sha1 = "439e35b0b36e2e5881738abc8857bd92ad6ff9a8"
uuid = "d38c429a-6771-53c6-b99e-75d170b6e991"
@ -434,6 +456,16 @@ git-tree-sha1 = "1a3f97f907e6dd8983b744d2642651bb162a3f7a"
uuid = "dc8bdbbb-1ca9-579f-8c36-e416f6a65cce"
version = "1.0.2"
[[deps.DSP]]
deps = ["Bessels", "FFTW", "IterTools", "LinearAlgebra", "Polynomials", "Random", "Reexport", "SpecialFunctions", "Statistics"]
git-tree-sha1 = "5989debfc3b38f736e69724818210c67ffee4352"
uuid = "717857b8-e6f2-59f4-9121-6e50c889abd2"
version = "0.8.4"
weakdeps = ["OffsetArrays"]
[deps.DSP.extensions]
OffsetArraysExt = "OffsetArrays"
[[deps.DataAPI]]
git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
@ -516,6 +548,12 @@ git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.9.5"
[[deps.Documenter]]
deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"]
git-tree-sha1 = "352b9a04e74edd16429aec79f033620cf8e780d4"
uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
version = "1.15.0"
[[deps.DotEnv]]
deps = ["PrecompileTools"]
git-tree-sha1 = "92e88cb68a5b10545234f46dfaeb2fa8a8a50c45"
@ -651,12 +689,6 @@ git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172"
uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
version = "0.8.5"
[[deps.FlameGraphs]]
deps = ["AbstractTrees", "Colors", "FileIO", "FixedPointNumbers", "IndirectArrays", "LeftChildRightSiblingTrees", "Profile"]
git-tree-sha1 = "0166baf81babb91cf78bfcc771d8e87c43d568df"
uuid = "08572546-2f56-4bcf-ba4e-bab62c3a3f89"
version = "1.1.0"
[[deps.Fontconfig_jll]]
deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Zlib_jll"]
git-tree-sha1 = "f85dac9a96a01087df6e3a749840015a0ca3817d"
@ -804,6 +836,24 @@ git-tree-sha1 = "6570366d757b50fabae9f4315ad74d2e40c0560a"
uuid = "59f7168a-df46-5410-90c8-f2779963d0ec"
version = "5.2.3+0"
[[deps.Git]]
deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"]
git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8"
uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
version = "1.5.0"
[[deps.Git_LFS_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"]
git-tree-sha1 = "bb8471f313ed941f299aa53d32a94ab3bee08844"
uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2"
version = "3.7.0+0"
[[deps.Git_jll]]
deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"]
git-tree-sha1 = "b6a684587ebe896d9f68ae777f648205940f0f70"
uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb"
version = "2.51.3+0"
[[deps.Glib_jll]]
deps = ["Artifacts", "GettextRuntime_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"]
git-tree-sha1 = "50c11ffab2a3d50192a228c313f05b5b5dc5acb2"
@ -885,6 +935,12 @@ git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec"
uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a"
version = "0.3.28"
[[deps.IOCapture]]
deps = ["Logging", "Random"]
git-tree-sha1 = "b6d6bfdd7ce25b0f9b2f6b3dd56b2673a66c8770"
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
version = "0.2.5"
[[deps.IfElse]]
git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1"
uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173"
@ -1236,6 +1292,11 @@ git-tree-sha1 = "a9eaadb366f5493a5654e843864c13d8b107548c"
uuid = "10f19ff3-798f-405d-979b-55457f8fc047"
version = "0.1.17"
[[deps.LazilyInitializedFields]]
git-tree-sha1 = "0f2da712350b020bc3957f269c9caad516383ee0"
uuid = "0e77f7df-68c5-4e49-93ce-4cd80f5598bf"
version = "1.3.0"
[[deps.LazyArtifacts]]
deps = ["Artifacts", "Pkg"]
uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
@ -1436,6 +1497,12 @@ deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
version = "1.11.0"
[[deps.MarkdownAST]]
deps = ["AbstractTrees", "Markdown"]
git-tree-sha1 = "465a70f0fc7d443a00dcdc3267a497397b8a3899"
uuid = "d0879d2d-cac2-40c8-9cee-1863dc0c7391"
version = "0.1.2"
[[deps.MathTeXEngine]]
deps = ["AbstractTrees", "Automa", "DataStructures", "FreeTypeAbstraction", "GeometryBasics", "LaTeXStrings", "REPL", "RelocatableFolders", "UnicodeFun"]
git-tree-sha1 = "a370fef694c109e1950836176ed0d5eabbb65479"
@ -1785,10 +1852,6 @@ deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
version = "1.11.0"
[[deps.Profile]]
uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
version = "1.11.0"
[[deps.ProgressMeter]]
deps = ["Distributed", "Printf"]
git-tree-sha1 = "fbb92c6c56b34e1a2c4c36058f68f332bec840e7"
@ -1872,6 +1935,12 @@ git-tree-sha1 = "4618ed0da7a251c7f92e869ae1a19c74a7d2a7f9"
uuid = "dee08c22-ab7f-5625-9660-a9af2021b33f"
version = "0.3.2"
[[deps.RegistryInstances]]
deps = ["LazilyInitializedFields", "Pkg", "TOML", "Tar"]
git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51"
uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3"
version = "0.1.0"
[[deps.RelocatableFolders]]
deps = ["SHA", "Scratch"]
git-tree-sha1 = "ffdaf70d81cf6ff22c2b6e733c900c3321cab864"
@ -2373,6 +2442,12 @@ git-tree-sha1 = "d1d9a935a26c475ebffd54e9c7ad11627c43ea85"
uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
version = "0.21.72"
[[deps.Wavelets]]
deps = ["DSP", "FFTW", "LinearAlgebra", "Reexport", "SpecialFunctions", "Statistics"]
git-tree-sha1 = "d0ec97a100abbe47a5e9a02361841da49cce6029"
uuid = "29a6e085-ba6d-5f35-a997-948ac2efa89a"
version = "0.10.1"
[[deps.Wayland_jll]]
deps = ["Artifacts", "EpollShim_jll", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll"]
git-tree-sha1 = "96478df35bbc2f3e1e791bc7a3d0eeee559e60e9"

7
Project.toml
View File

@ -7,12 +7,13 @@ Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
CodecBase = "6c391c72-fb7b-5838-ba82-7cfb1bcfecbf"
ColorSchemes = "35d6a980-a343-548e-a6ea-1d62b119f2f4"
Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
ContinuousWavelets = "96eb917e-2868-4417-9cb6-27e7ff17528f"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
FlameGraphs = "08572546-2f56-4bcf-ba4e-bab62c3a3f89"
GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a"
Genie = "c43c736e-a2d1-11e8-161f-af95117fbd1e"
GenieFramework = "a59fdf5c-6bf0-4f5d-949c-a137c9e2f353"
@ -25,6 +26,8 @@ ImageFiltering = "6a3955dd-da59-5b1f-98d4-e7296123deb5"
ImageMorphology = "787d08f9-d448-5407-9aad-5290dd7ab264"
ImageSegmentation = "80713f31-8817-5129-9cf8-209ff8fb23e1"
Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0"
Interpolations = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59"
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
Libz = "2ec943e9-cfe8-584d-b93d-64dcb6d567b7"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@ -47,7 +50,7 @@ Accessors = "0.1"
Base64 = "1.11"
CSV = "0.10"
CairoMakie = "0.13"
ColorSchemes = "3.31"
ColorSchemes = "3.30"
Colors = "0.12"
DataFrames = "1.7"
Dates = "1.11"

33
app.jl
View File

@ -213,6 +213,7 @@ end
# == Batch Processing & Registry Variables ==
@private registry_init_done = false
@in refetch_folders = false
@in selected_files = String[]
@in available_folders = String[]
@in image_available_folders = String[]
@ -1919,6 +1920,37 @@ end
end
end
@onbutton refetch_folders begin
# Re-load registry and update folder lists
registry = load_registry(registry_path)
all_folders = sort(collect(keys(registry)), lt=natural)
img_folders = filter(folder -> get(get(registry, folder, Dict()), "is_imzML", false), all_folders)
available_folders = deepcopy(all_folders)
image_available_folders = deepcopy(img_folders)
# For q-selects using image_available_folders
if !isempty(image_available_folders)
first_img_folder = first(image_available_folders)
if isempty(selected_folder_main)
selected_folder_main = first_img_folder
end
if isempty(selected_folder_compare_left)
selected_folder_compare_left = first_img_folder
end
if isempty(selected_folder_compare_right)
selected_folder_compare_right = first_img_folder
end
end
# For q-selects using available_folders
if !isempty(available_folders)
if isempty(selected_folder_metadata)
selected_folder_metadata = first(available_folders)
end
end
end
@mounted watchplots()
@onchange isready @time begin
@ -1967,7 +1999,6 @@ end
image_available_folders = deepcopy(img_folders)
println("UI lists updated. All: $(length(available_folders)), Images: $(length(image_available_folders))")
catch e
@warn "Registry synchronization failed: $e"
available_folders = []

392
app.jl.html
View File

@ -19,7 +19,7 @@
<div class="text-h6">imzML & mzML Data Pre-Treatment</div>
<div class="row items-center">
<q-input standout="custom-standout" class="q-ma-sm cursor-pointer col" v-model="full_route" readonly
:label="batch_file_count > 0 ? batch_file_count + ' file(s) in batch' : 'Select an imzML file'"
:label="batch_file_count > 0 ? batch_file_count + ' file(s) in batch' : 'Select an imzMl / mzML file'"
v-on:click="btnSearch=true">
<template v-slot:append>
<q-icon name="search" v-on:click="btnSearch=true" class="cursor-pointer" />
@ -29,6 +29,22 @@
<q-btn class="q-ma-sm" icon="clear" v-on:click="clear_batch_btn=true" :disable="batch_file_count === 0"
label="Clear"></q-btn>
</div>
<q-list bordered separator v-if="selected_files.length > 0">
<q-item v-for="(file, index) in selected_files" :key="index">
<q-item-section>
{{ file }}
</q-item-section>
<q-item-section side>
<q-btn flat round icon="delete" size="sm" v-on:click="selected_files.splice(index, 1)"></q-btn>
</q-item-section>
</q-item>
</q-list>
<q-input standout="custom-standout" class="q-ma-sm cursor-pointer col" v-model="full_route_cal" readonly
label="Select a calibration imzMl / mzML file" v-on:click="btnSearchCal=true">
<template v-slot:append>
<q-icon name="search" v-on:click="btnSearchCal=true" class="cursor-pointer" />
</template>
</q-input>-
<br>
<br>
<q-tabs v-model="pre_tab" dense class="text-grey" indicator-color="primary" align="justify">
@ -40,10 +56,176 @@
<q-tab name="peak" label="Peak Detection"></q-tab>
<q-tab name="binning" label="Binning"></q-tab>
</q-tabs>
<q-separator></q-separator>
<q-separator />
<q-tab-panels v-model="pre_tab" animated>
<q-tab-panel name="stabilization">
<p>HI!</p>
<q-tab-panel name="stabilization" class="q-pa-md">
<q-card class="q-mb-md">
<q-card-section>
<div class="text-h6">Method</div>
</q-card-section>
<q-card-section>
<q-radio v-model="stabilization_method" val="sqrt" label="SQRT" /><br>
<q-radio v-model="stabilization_method" val="log" label="LOG" /><br>
<q-radio v-model="stabilization_method" val="log2" label="LOG 2" /><br>
<q-radio v-model="stabilization_method" val="log10" label="LOG 10" /><br>
</q-card-section>
</q-card>
<div class="row justify-end">
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="play_arrow" v-on:click="acceptStab=true"
padding="lg" label="Accept" />
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="clear" v-on:click="undoPrep=true"
padding="lg" label="Undo" />
</div>
</q-tab-panel>
<q-tab-panel name="smoothing">
<q-card class="q-mb-md">
<q-card-section>
<div class="text-h6">Method</div>
</q-card-section>
<q-card-section>
<q-radio v-model="smoothing_method" val="sg" label="Savitzky-Golay" /><br>
<q-radio v-model="smoothing_method" val="ma" label="Moving Average" /><br>
</q-card-section>
</q-card>
<q-card>
<q-card-section>
<div class="text-h6">Parameters</div>
</q-card-section>
<q-card-section>
<q-input standout="custom-standout" label="Half Window Size" type="number"
v-model.number="smoothing_window"></q-input>
</q-card-section>
</q-card>
<div class="row justify-end q-mt-md">
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="play_arrow" v-on:click="acceptSmoo=true"
padding="lg" label="Accept" />
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="clear" v-on:click="undoPrep=true"
padding="lg" label="Undo" />
</div>
</q-tab-panel>
<q-tab-panel name="baseline">
<q-card class="q-mb-md">
<q-card-section>
<div class="text-h6">Method</div>
</q-card-section>
<q-card-section>
<q-radio v-model="baseline_method" val="snip" label="SNIP" /><br>
<q-radio v-model="baseline_method" val="tophat" label="TOP HAT" /><br>
<q-radio v-model="baseline_method" val="convex_hull" label="CONVEX HULL" /><br>
<q-radio v-model="baseline_method" val="median" label="MEDIAN" /><br>
</q-card-section>
</q-card>
<q-card>
<q-card-section>
<div class="text-h6">Parameters</div>
</q-card-section>
<q-card-section>
<q-input standout="custom-standout" label="Iterations" type="number"
v-model.number="baseline_iterations"></q-input>
</q-card-section>
</q-card>
<div class="row justify-end q-mt-md">
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="play_arrow" v-on:click="acceptBase=true"
padding="lg" label="Accept" />
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="clear" v-on:click="undoPrep=true"
padding="lg" label="Undo" />
</div>
</q-tab-panel>
<q-tab-panel name="calibration">
<q-card class="q-mb-md">
<q-card-section>
<div class="text-h6">Method</div>
</q-card-section>
<q-card-section>
<q-radio v-model="calibration_method" val="tic" label="TIC" /><br>
<q-radio v-model="calibration_method" val="pqn" label="PQN" /><br>
<q-radio v-model="calibration_method" val="median" label="MEDIAN" /><br>
</q-card-section>
</q-card>
<div class="row justify-end q-mt-md">
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="play_arrow" v-on:click="acceptCali=true"
padding="lg" label="Accept" />
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="clear" v-on:click="undoPrep=true"
padding="lg" label="Undo" />
</div>
</q-tab-panel>
<q-tab-panel name="warping">
<q-card class="q-mb-md">
<q-card-section>
<div class="text-h6">Method</div>
</q-card-section>
<q-card-section>
<q-radio v-model="warping_method" val="snip" label="SNIP" /><br>
<q-radio v-model="warping_method" val="tophat" label="TOP HAT" /><br>
<q-radio v-model="warping_method" val="convex_hull" label="CONVEX HULL" /><br>
<q-radio v-model="warping_method" val="median" label="MEDIAN" /><br>
</q-card-section>
</q-card>
<q-card>
<q-card-section>
<div class="text-h6">Parameters</div>
</q-card-section>
<q-card-section>
<q-input standout="custom-standout" label="Iterations" type="number"
v-model.number="warping_iterations"></q-input>
</q-card-section>
</q-card>
<div class="row justify-end q-mt-md">
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="play_arrow" v-on:click="acceptWarp=true"
padding="lg" label="Accept" />
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="clear" v-on:click="undoPrep=true"
padding="lg" label="Undo" />
</div>
</q-tab-panel>
<q-tab-panel name="peak">
<p>Detect if profile or centroid to determine which elements to show.</p>
<q-card class="q-mb-md">
<q-card-section>
<div class="text-h6">Method</div>
</q-card-section>
<q-card-section>
<q-radio v-model="peak_method" val="mad" label="MAD" /><br>
<q-radio v-model="peak_method" val="super_smoother" label="SUPER SMOOTHER" /><br>
</q-card-section>
</q-card>
<q-card>
<q-card-section>
<div class="text-h6">Parameters</div>
</q-card-section>
<q-card-section>
<q-input standout="custom-standout" label="Signal to Noise" type="number"
v-model.number="peak_snr"></q-input>
<q-input standout="custom-standout" label="Half Window Size" type="number" class="q-mt-md"
v-model.number="peak_window"></q-input>
<q-input standout="custom-standout" label="Intensity Threshold" type="number" class="q-mt-md"
v-model.number="peak_threshold"></q-input>
</q-card-section>
</q-card>
<div class="row justify-end q-mt-md">
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="play_arrow" v-on:click="acceptPeak=true"
padding="lg" label="Accept" />
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="clear" v-on:click="undoPrep=true"
padding="lg" label="Undo" />
</div>
</q-tab-panel>
<q-tab-panel name="binning">
<q-card>
<q-card-section>
<div class="text-h6">Parameters</div>
</q-card-section>
<q-card-section>
<q-input standout="custom-standout" label="Tolerance" type="number"
v-model.number="binning_tolerance"></q-input>
<q-input standout="custom-standout" label="Frequency Threshold" type="number" class="q-mt-md"
v-model.number="binning_threshold"></q-input>
</q-card-section>
</q-card>
<div class="row justify-end q-mt-md">
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="play_arrow" v-on:click="acceptBinn=true"
padding="lg" label="Accept" />
<q-btn :loading="progressPrep" class="q-ma-sm btn-style" icon="clear" v-on:click="undoPrep=true"
padding="lg" label="Undo" />
</div>
</q-tab-panel>
</q-tab-panels>
</q-tab-panel>
@ -56,7 +238,7 @@
</p>
<div class="row items-center">
<q-input standout="custom-standout" class="q-ma-sm cursor-pointer col" v-model="full_route" readonly
:label="batch_file_count > 0 ? batch_file_count + ' file(s) in batch' : 'Select an imzML file'"
:label="batch_file_count > 0 ? batch_file_count + ' file(s) in batch' : 'Select an imzMl / mzML file'"
v-on:click="btnSearch=true">
<template v-slot:append>
<q-icon name="search" v-on:click="btnSearch=true" class="cursor-pointer" />
@ -171,10 +353,9 @@
</q-btn>
<q-btn icon="zoom_out_map" class="q-ma-sm on-right btn-style" v-on:click="compareBtn=true" padding="sm"
label="Compare"></q-btn>
<!--<q-btn class="q-ma-sm btn-style" :disable="btnMetadataDisable"
v-on:click="showMetadataBtn=true" label="Show Metadata"></q-btn>-->
<q-btn class="q-ma-sm btn-style" icon="edit" label="Mask Editor" href="/mask"></q-btn>
<q-btn class="q-ma-sm btn-style" v-on:click="showMetadataBtn=true" label="Show Metadata"></q-btn>
<q-btn class="q-ma-sm btn-style" icon="dashboard" v-on:click="showMetadataBtn=true"
label="Show Metadata"></q-btn>
<div class="q-pa-md row items-center" v-show="progress">
<q-spinner color="primary" size="2em" class="q-mr-sm"></q-spinner>
<div class="text-caption">{{ progress_message }}</div>
@ -238,7 +419,6 @@
<div class="text-h6">mzML to imzML Converter</div>
<p>Select the .mzML file and the corresponding .txt synchronization file to convert them into an .imzML/.ibd
pair.</p>
<q-input standout="custom-standout" class="q-ma-sm cursor-pointer" v-model="mzml_full_route" readonly
label="Select your .mzML file" v-on:click="btnSearchMzml=true">
<template v-slot:append>
@ -268,96 +448,118 @@
<div class="row col-6">
<!-- Right DIV -->
<div id="intDivStyle-right" class="st-col col-12 col-sm st-module">
<st-tabs id="tabHeader-main" :ids="tabIDs" :labels="tabLabels" v-model="selectedTab" no-arrows></st-tabs>
<q-tab-panels v-model="selectedTab">
<q-tab-panel name="tab0">
<!-- Content for Tab 0 -->
<h6>Image visualizer</h6>
<div class="row items-center">
<q-select v-model="selected_folder_main" :options="image_available_folders" label="Select Dataset"
class="q-ma-sm" style="min-width: 200px;"></q-select>
<q-space></q-space>
<q-btn icon="arrow_back" class="q-my-sm btn-style" v-on:click="imgMinus=true"></q-btn>
<q-btn icon="arrow_forward" class="q-my-sm on-right btn-style" v-on:click="imgPlus=true"></q-btn>
</div>
<!-- Image manager -->
<div id="image-container-normal" class="row st-col col-12">
<div class="col-10 q-pa-none q-ma-none">
<plotly id="plotImg" :data="plotdataImg" :layout="plotlayoutImg" class="q-pa-none q-ma-none sync_data"
@click="data_click"></plotly>
<div v-if="left_tab === 'pre_treatment'">
<div class="text-h6 q-mb-md">Spectrum View</div>
<q-card class="q-mb-md">
<q-card-section>
<div class="text-subtitle1">Before Preprocessing</div>
</q-card-section>
<q-card-section>
<plotly id="plotSpectraBefore" :data="plotdata" :layout="plotlayout" class="q-pa-none q-ma-none"></plotly>
</q-card-section>
</q-card>
<q-card>
<q-card-section>
<div class="text-subtitle1">After Preprocessing</div>
</q-card-section>
<q-card-section>
<plotly id="plotSpectraAfter" :data="plotdata_after" :layout="plotlayout_after" class="q-pa-none q-ma-none">
</plotly>
</q-card-section>
</q-card>
</div>
<div v-else>
<st-tabs id="tabHeader-main" :ids="tabIDs" :labels="tabLabels" v-model="selectedTab" no-arrows></st-tabs>
<q-tab-panels v-model="selectedTab">
<q-tab-panel name="tab0">
<!-- Content for Tab 0 -->
<h6>Image visualizer</h6>
<div class="row items-center">
<q-select v-model="selected_folder_main" :options="image_available_folders" label="Select Dataset"
class="q-ma-sm" style="min-width: 200px;" v-on:focus="refetch_folders = true"></q-select>
<q-space></q-space>
<q-btn icon="arrow_back" class="q-my-sm btn-style" v-on:click="imgMinus=true"></q-btn>
<q-btn icon="arrow_forward" class="q-my-sm on-right btn-style" v-on:click="imgPlus=true"></q-btn>
</div>
<div class="col-2 q-pa-none q-ma-none">
<q-img id="colorbar-normal" class="q-ma-none q-pa-none" :src="colorbar"></q-img>
<!-- Image manager -->
<div id="image-container-normal" class="row st-col col-12">
<div class="col-10 q-pa-none q-ma-none">
<plotly id="plotImg" :data="plotdataImg" :layout="plotlayoutImg" class="q-pa-none q-ma-none sync_data"
@click="data_click"></plotly>
</div>
<div class="col-2 q-pa-none q-ma-none">
<q-img id="colorbar-normal" class="q-ma-none q-pa-none" :src="colorbar"></q-img>
</div>
</div>
</div>
<p v-html="msgimg"></p>
</q-tab-panel>
<p v-html="msgimg"></p>
</q-tab-panel>
<q-tab-panel name="tab1">
<!-- Content for Tab 1 -->
<h6>TrIQ visualizer</h6>
<div class="row items-center">
<q-select v-model="selected_folder_main" :options="image_available_folders" label="Select Dataset"
class="q-ma-sm" style="min-width: 200px;"></q-select>
<q-space></q-space>
<q-btn icon="arrow_back" class="q-my-sm btn-style" v-on:click="imgMinusT=true"></q-btn>
<q-btn icon="arrow_forward" class="q-my-sm on-right btn-style" v-on:click="imgPlusT=true"></q-btn>
</div>
<!-- Triq Image manager -->
<div id="image-container-triq" class="row st-col col-12">
<div class="col-10 q-pa-none q-ma-none ">
<plotly id="plotImgT" :data="plotdataImgT" :layout="plotlayoutImgT" class="q-pa-none q-ma-none sync_data"
@click="data_click"></plotly>
<q-tab-panel name="tab1">
<!-- Content for Tab 1 -->
<h6>TrIQ visualizer</h6>
<div class="row items-center">
<q-select v-model="selected_folder_main" :options="image_available_folders" label="Select Dataset"
class="q-ma-sm" style="min-width: 200px;" v-on:focus="refetch_folders = true"></q-select>
<q-space></q-space>
<q-btn icon="arrow_back" class="q-my-sm btn-style" v-on:click="imgMinusT=true"></q-btn>
<q-btn icon="arrow_forward" class="q-my-sm on-right btn-style" v-on:click="imgPlusT=true"></q-btn>
</div>
<div class="col-2 q-pa-none q-ma-none ">
<q-img id="colorbar-triq" class="q-ma-none q-pa-none" :src="colorbarT"></q-img>
<!-- Triq Image manager -->
<div id="image-container-triq" class="row st-col col-12">
<div class="col-10 q-pa-none q-ma-none ">
<plotly id="plotImgT" :data="plotdataImgT" :layout="plotlayoutImgT"
class="q-pa-none q-ma-none sync_data" @click="data_click"></plotly>
</div>
<div class="col-2 q-pa-none q-ma-none ">
<q-img id="colorbar-triq" class="q-ma-none q-pa-none" :src="colorbarT"></q-img>
</div>
</div>
</div>
<p v-html="msgtriq"></p>
</q-tab-panel>
<p v-html="msgtriq"></p>
</q-tab-panel>
<q-tab-panel name="tab2">
<div class="row items-center">
<q-select v-model="selected_folder_main" :options="available_folders" label="Select Dataset" class="q-ma-sm"
style="min-width: 200px;"></q-select>
<q-btn-dropdown class="q-ma-sm btn-style" :loading="progressSpectraPlot" :disable="btnSpectraDisable"
label="Generate Spectra" icon="play_arrow">
<template v-slot:loading>
<q-spinner-hourglass class="on-left" />
Loading plot
</template>
<q-tab-panel name="tab2">
<div class="row items-center">
<q-select v-model="selected_folder_main" :options="available_folders" label="Select Dataset"
class="q-ma-sm" style="min-width: 200px;" v-on:focus="refetch_folders = true"></q-select>
<q-btn-dropdown class="q-ma-sm btn-style" :loading="progressSpectraPlot" :disable="btnSpectraDisable"
label="Generate Spectra" icon="play_arrow">
<template v-slot:loading>
<q-spinner-hourglass class="on-left" />
Loading plot
</template>
<q-list>
<q-item clickable v-close-popup v-on:click="createMeanPlot=true">
<q-item-section>
<q-item-label>Mean spectrum plot</q-item-label>
</q-item-section>
</q-item>
<q-item clickable v-close-popup v-on:click="createSumPlot=true">
<q-item-section>
<q-item-label>Sum Spectrum plot</q-item-label>
</q-item-section>
</q-item>
<q-item clickable v-close-popup v-on:click="createXYPlot=true">
<q-item-section>
<q-item-label>Spectrum plot (X,Y)</q-item-label>
</q-item-section>
</q-item>
</q-list>
</q-btn-dropdown>
</div>
<plotly id="plotSpectra" :data="plotdata" :layout="plotlayout" class="q-pa-none q-ma-none"></plotly>
</q-tab-panel>
<q-list>
<q-item clickable v-close-popup v-on:click="createMeanPlot=true">
<q-item-section>
<q-item-label>Mean spectrum plot</q-item-label>
</q-item-section>
</q-item>
<q-item clickable v-close-popup v-on:click="createSumPlot=true">
<q-item-section>
<q-item-label>Sum Spectrum plot</q-item-label>
</q-item-section>
</q-item>
<q-item clickable v-close-popup v-on:click="createXYPlot=true">
<q-item-section>
<q-item-label>Spectrum plot (X,Y)</q-item-label>
</q-item-section>
</q-item>
</q-list>
</q-btn-dropdown>
</div>
<plotly id="plotSpectra" :data="plotdata" :layout="plotlayout" class="q-pa-none q-ma-none"></plotly>
</q-tab-panel>
<q-tab-panel name="tab3">
<!-- Content for Tab 3 -->
<plotly id="plotTopo" :data="plotdataC" :layout="plotlayoutC" class="q-pa-none q-ma-none"></plotly>
</q-tab-panel>
<q-tab-panel name="tab4">
<!-- Content for Tab 4 -->
<plotly id="plot3d" :data="plotdata3d" :layout="plotlayout3d" class="q-pa-none q-ma-none"></plotly>
</q-tab-panel>
</q-tab-panels>
<q-tab-panel name="tab3">
<!-- Content for Tab 3 -->
<plotly id="plotTopo" :data="plotdataC" :layout="plotlayoutC" class="q-pa-none q-ma-none"></plotly>
</q-tab-panel>
<q-tab-panel name="tab4">
<!-- Content for Tab 4 -->
<plotly id="plot3d" :data="plotdata3d" :layout="plotlayout3d" class="q-pa-none q-ma-none"></plotly>
</q-tab-panel>
</q-tab-panels>
</div>
</div>
</div>
</div>
@ -395,7 +597,8 @@
<div class="col-6">
<div class="row items-center">
<q-select v-model="selected_folder_compare_left" :options="image_available_folders"
label="Select Left Dataset" class="q-ma-sm" style="min-width: 200px;"></q-select>
label="Select Left Dataset" class="q-ma-sm" style="min-width: 200px;"
v-on:focus="refetch_folders = true"></q-select>
<q-space></q-space>
<st-tabs id="tabHeaderCompareLeft" :ids="CompTabIDsLeft" :labels="CompTabLabelsLeft"
v-model="CompSelectedTabLeft"></st-tabs>
@ -464,7 +667,8 @@
<div class="col-6">
<div class="row items-center">
<q-select v-model="selected_folder_compare_right" :options="image_available_folders"
label="Select Right Dataset" class="q-ma-sm" style="min-width: 200px;"></q-select>
label="Select Right Dataset" class="q-ma-sm" style="min-width: 200px;"
v-on:focus="refetch_folders = true"></q-select>
<q-space></q-space>
<st-tabs id="tabHeaderCompareRight" :ids="CompTabIDsRight" :labels="CompTabLabelsRight"
v-model="CompSelectedTabRight"></st-tabs>
@ -561,7 +765,7 @@
<div class="text-h6">Dataset Summary</div>
<q-space />
<q-select v-model="selected_folder_metadata" :options="available_folders" label="Select Dataset" class="q-ma-sm"
style="min-width: 250px;" standout="custom-standout"></q-select>
style="min-width: 250px;" standout="custom-standout" v-on:focus="refetch_folders = true"></q-select>
</q-card-section>
<q-card-section class="q-pt-none">

6
julia_imzML_visual.jl
View File

@ -666,10 +666,10 @@ function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int,
# Downsample for plotting performance
mz_down, int_down = MSI_src.downsample_spectrum(mz, intensity)
trace = if spectrum_mode == MSI_src.PROFILE
PlotlyBase.scatter(x=mz_down, y=int_down, mode="lines", marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
else
trace = if spectrum_mode == MSI_src.CENTROID
PlotlyBase.stem(x=mz_down, y=int_down, marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
else
PlotlyBase.scatter(x=mz_down, y=int_down, mode="lines", marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
end
plotdata = [trace]

18
mask.jl
View File

@ -151,6 +151,7 @@ end
@out available_folders = String[]
@out image_available_folders = String[]
@private registry_init_done = false
@in refetch_folders = false
@out imgInt = "" # Path to current slice
@out current_msi = ""
@ -711,6 +712,23 @@ end
end
end
@onbutton refetch_folders begin
# First, re-run the logic to populate the folder list
registry = load_registry(registry_path)
all_folders = sort(collect(keys(registry)), lt=natural)
img_folders = filter(folder -> get(get(registry, folder, Dict()), "is_imzML", false), all_folders)
available_folders = deepcopy(all_folders)
image_available_folders = deepcopy(img_folders)
# Apply the logic to select the first item if none is selected
if !isempty(image_available_folders) && isempty(selected_folder_main)
# This ensures the list is up-to-date
image_available_folders = deepcopy(img_folders)
selected_folder_main = first(image_available_folders)
end
end
@onchange isready begin
if isready && !registry_init_done
sleep(1.0) # Give frontend time to initialize

2
mask.jl.html
View File

@ -15,7 +15,7 @@
<div class="text-subtitle1 q-mt-md">Step 1: Select Slice</div>
<div class="row items-center">
<q-select v-model="selected_folder_main" :options="image_available_folders" label="Select Dataset"
class="q-ma-sm col" standout="custom-standout" :disable="is_editing_mask"></q-select>
class="q-ma-sm col" standout="custom-standout" :disable="is_editing_mask" v-on:focus="refetch_folders = true"></q-select>
</div>
<p class="text-center" v-html="msgimg"></p>

33
public/css/genieapp.css
View File

@ -45,9 +45,10 @@
color: #009f90;
}
#intDivStyle-left {
#intDivStyle-left, #intDivStyle-right {
border-radius: 10px;
padding: 10px;
background-color: #fdfdfd; /* Add a background color to distinguish from the outer div */
}
/* Tab styling to match your theme */
@ -81,13 +82,13 @@
font-size: 0.9rem;
}
#intDivStyle-left .q-tab-panels {
height: 700px; /* Set this to accommodate your tallest content */
#intDivStyle-left .q-tab-panels, #intDivStyle-right .q-tab-panels {
/* Removed fixed height */
}
#intDivStyle-left .q-tab-panel {
height: 100%;
overflow-y: auto; /* Add scroll if content overflows */
#intDivStyle-left .q-tab-panel, #intDivStyle-right .q-tab-panel {
height: auto; /* Let content define height */
overflow-y: hidden; /* Remove scrollbar */
}
.pixelated-plot svg image {
@ -98,4 +99,22 @@
*{
font-family: 'Roboto', 'Lato', sans-serif;
}
}
.q-card {
border-radius: 8px !important;
box-shadow: 0 2px 4px rgba(0,0,0,0.1) !important;
margin-bottom: 16px !important;
}
.q-card__section--dark {
background: #f7f7f7 !important;
}
.q-radio__label {
font-size: 1rem !important;
}
.q-option-group > div {
margin-bottom: 8px;
}

64
src/BloomFilters.jl
View File

@ -37,9 +37,20 @@ mutable struct StreamingBloomFilter
end
"""
BloomFilter(expected_elements::Int, false_positive_rate::Float64=0.01; seed::UInt64=0x12345678)
BloomFilter{T}(expected_elements::Int, false_positive_rate::Float64=0.01; kwargs...) -> Return type
Creates a Bloom filter optimized for the expected number of elements and desired false positive rate.
# Arguments
- `expected_elements::Int`: Argument description
- `false_positive_rate::Float64`: Argument description
(**Default**: `0.01`)
# Keywords
- `seed::Union{UInt32,UInt64}`: Keyword description
(**Default**: `0x12345678`)
"""
function BloomFilter{T}(expected_elements::Int, false_positive_rate::Float64=0.01; seed::Union{UInt32,UInt64}=0x12345678) where T
# Convert seed to UInt64 for consistency
@ -56,9 +67,12 @@ end
"""
optimal_bit_size(n::Int, p::Float64) -> Int
Calculates the optimal number of bits for a Bloom filter given:
- n: expected number of elements
- p: desired false positive rate
Calculates the optimal number of bits for a Bloom filter
# Arguments
- `n::Int`: Expected number of elements
- `p::Float64`: Desired false positive rate
"""
function optimal_bit_size(n::Int, p::Float64)::Int
if p <= 0.0 || p >= 1.0
@ -73,6 +87,11 @@ end
optimal_hash_count(n::Int, m::Int) -> Int
Calculates the optimal number of hash functions for a Bloom filter.
# Arguments
- `n::Int`: Expected number of elements
- `p::Float64`: Desired false positive rate
"""
function optimal_hash_count(n::Int, m::Int)::Int
if n <= 0 || m <= 0
@ -109,6 +128,11 @@ end
Base.push!(bf::BloomFilter{T}, item::T)
Adds an element to the Bloom filter.
# Arguments
- `bf::BloomFilter{T}`: Argument description
- `item::T`: Argument description
"""
function Base.push!(bf::BloomFilter{T}, item::T) where T
hashes = hash_functions(item, bf.hash_count, bf.size, bf.seed)
@ -169,27 +193,39 @@ function false_positive_rate(bf::BloomFilter)::Float64
end
"""
fill_ratio(bf::BloomFilter) -> Float64
fill_ratio(bf::BloomFilter)::Float64 return count(bf.bits) / length(bf.bits) end -> Return type
Returns the fraction of bits that are set to 1.
# Arguments
- `bf::BloomFilter`: Argument description
"""
function fill_ratio(bf::BloomFilter)::Float64
return count(bf.bits) / length(bf.bits)
end
"""
is_empty(bf::BloomFilter) -> Bool
is_empty(bf::BloomFilter)::Bool return bf.count == 0 end -> Return type
Checks if the Bloom filter is empty (no elements added).
# Arguments
- `bf::BloomFilter`: Argument description
"""
function is_empty(bf::BloomFilter)::Bool
return bf.count == 0
end
"""
reset!(bf::BloomFilter)
reset!(bf::BloomFilter) fill!(bf.bits, false) bf.count = 0 return bf end -> Return type
Clears the Bloom filter, removing all elements.
# Arguments
- `bf::BloomFilter`: Argument description
"""
function reset!(bf::BloomFilter)
fill!(bf.bits, false)
@ -198,6 +234,20 @@ function reset!(bf::BloomFilter)
end
# Specialized constructor for empty Bloom filters
"""
BloomFilter{T}(; kwargs...) -> Return type
Description of the function
# Keywords
- `size::Int`: Keyword description
(**Default**: `100`)
- `hash_count::Int`: Keyword description
(**Default**: `3`)
- `seed::Union{UInt32,UInt64}`: Keyword description
(**Default**: `0x12345678`)
"""
function BloomFilter{T}(;size::Int=100, hash_count::Int=3, seed::Union{UInt32,UInt64}=0x12345678) where T
seed_uint64 = UInt64(seed)
bits = falses(size)

6
src/MSIData.jl
View File

@ -420,6 +420,12 @@ function read_binary_vector(data::MSIData, io::IO, asset::SpectrumAsset)
return out_array
end
function read_binary_vector(data::MSIData, ts_handle::ThreadSafeFileHandle, asset::SpectrumAsset)
lock(ts_handle.lock) do
return read_binary_vector(data, ts_handle.handle, asset)
end
end
# Overload for different source types
"""
read_spectrum_from_disk(source::ImzMLSource, meta::SpectrumMetadata)

4
src/MSI_src.jl
View File

@ -28,8 +28,12 @@ export FeatureMatrix,
snip_baseline,
tic_normalize,
pqn_normalize,
median_normalize,
detect_peaks_profile,
detect_peaks_wavelet,
detect_peaks_centroid,
align_peaks_lowess,
find_calibration_peaks,
bin_peaks,
plot_stage_spectrum,
calculate_ppm_error,

927
src/Preprocessing.jl
View File

File diff suppressed because it is too large Load Diff

446
src/PreprocessingVersioning.jl Normal file
View File

@ -0,0 +1,446 @@
# src/PreprocessingVersioning.jl
"""
This module provides a versioning helper functions in our data preprocessing, module
inspired by the functionality of the R package MALDIquant. It includes
functions for quality control, intensity transformation, smoothing, baseline correction,
normalization, peak picking, alignment, and feature matrix generation.
This module is unfinished and may not end up in the release version.
"""
# =============================================================================
# Dependencies
# =============================================================================
using UUIDs, JLD2, CodecBase # For preprocessing versioning
# =============================================================================
# Data Structures
# =============================================================================
"""
VersionedSpectralData
A struct to hold a snapshot of spectral data at a specific point in the
preprocessing workflow. Each transformation creates a new instance of this
struct, forming a history of all operations.
# Fields
- `version_id::String`: A unique identifier for this version of the data.
- `parent_id::String`: The identifier of the version from which this one was derived.
- `spectra::Vector`: The spectral data itself, typically a vector of `(mz, intensity)` tuples.
- `processing_step::String`: A description of the operation that created this version (e.g., "smooth", "baseline").
- `timestamp::DateTime`: The time at which this version was created.
- `parameters::Dict`: A dictionary of the parameters used in the processing step.
"""
struct VersionedSpectralData
version_id::String
parent_id::String
spectra::Vector
processing_step::String
timestamp::DateTime
parameters::Dict
end
"""
MSISession
Manages the state of a preprocessing session, including the history of all
data versions and the current working version.
# Fields
- `session_id::String`: A unique identifier for the entire session.
- `original_data::VersionedSpectralData`: The initial, unprocessed spectral data.
- `history::Vector{VersionedSpectralData}`: A chronological list of all data versions created during the session.
- `current_version::VersionedSpectralData`: The version of the data currently being worked on or displayed.
- `processing_steps::Vector{String}`: A human-readable log of the processing steps applied.
"""
mutable struct MSISession
session_id::String
original_data::VersionedSpectralData
history::Vector{VersionedSpectralData}
current_version::VersionedSpectralData
processing_steps::Vector{String}
end
# =============================================================================
# Session Management
# =============================================================================
"""
initialize_session(raw_spectra::Vector, session_name::String) -> MSISession
Creates a new preprocessing session from raw spectral data.
# Arguments
- `raw_spectra::Vector`: A vector of `(mz, intensity)` tuples representing the initial dataset.
- `session_name::String`: A name for the session.
# Returns
- `MSISession`: A new session object initialized with the provided data.
"""
function initialize_session(raw_spectra::Vector, session_name::String)::MSISession
original = VersionedSpectralData(
string(uuid4()),
"root",
raw_spectra,
"Original Data",
now(),
Dict()
)
return MSISession(session_name, original, [original], original, [])
end
"""
get_processing_history(session::MSISession) -> Vector{String}
Returns a list of the processing steps applied during the session.
"""
get_processing_history(session::MSISession) = [v.processing_step for v in session.history]
"""
undo_last_step!(session::MSISession)
Reverts the session to the state before the last processing step was applied.
This modifies the session in place.
"""
function undo_last_step!(session::MSISession)
if length(session.history) > 1
pop!(session.history)
pop!(session.processing_steps)
session.current_version = last(session.history)
end
return session
end
"""
revert_to_version!(session::MSISession, version_id::String)
Reverts the session to a specific version in its history, discarding all
subsequent changes. This modifies the session in place.
"""
function revert_to_version!(session::MSISession, version_id::String)
target_idx = findfirst(v -> v.version_id == version_id, session.history)
if !isnothing(target_idx)
session.history = session.history[1:target_idx]
session.current_version = session.history[end]
# Also truncate the descriptive processing steps
num_steps_to_keep = max(0, target_idx - 1)
session.processing_steps = session.processing_steps[1:num_steps_to_keep]
end
return session
end
# =============================================================================
# Versioned Preprocessing
# =============================================================================
"""
apply_processing_step(spectra::Vector, step::Symbol, params::Dict) -> Vector
A dispatcher that applies a single, specified preprocessing function to a set of spectra.
This is the core function called by the versioned workflow.
# Arguments
- `spectra::Vector`: The input spectral data.
- `step::Symbol`: The symbol representing the processing step (e.g., `:smooth`, `:baseline`).
- `params::Dict`: A dictionary of parameters for the step.
# Returns
- `Vector`: The processed spectral data.
"""
function apply_processing_step(spectra::Vector, step::Symbol, params::Dict)
processed = deepcopy(spectra)
params_sym = Dict(Symbol(k) => v for (k,v) in params) # Ensure keys are symbols
if step === :transform
for i in eachindex(processed)
mz, y = processed[i]
processed[i] = (mz, transform_intensity(y; params_sym...))
end
elseif step === :smooth
for i in eachindex(processed)
mz, y = processed[i]
processed[i] = (mz, smooth_spectrum(y; params_sym...))
end
elseif step === :baseline
for i in eachindex(processed)
mz, y = processed[i]
baseline = snip_baseline(y; params_sym...)
processed[i] = (mz, max.(0.0, y .- baseline))
end
elseif step === :normalize
mode = get(params, :normalize_method, :tic)
if mode === :pqn
matrix = hcat([float.(p[2]) for p in processed]...)
matrix_norm = pqn_normalize(matrix)
for i in eachindex(processed)
mz, _ = processed[i]
processed[i] = (mz, view(matrix_norm, :, i))
end
else # :tic or :median
norm_func = (mode === :tic) ? tic_normalize : median_normalize
for i in eachindex(processed)
mz, y = processed[i]
processed[i] = (mz, norm_func(y))
end
end
elseif step === :peaks
peak_results = Vector{Tuple{Vector{Float64},Vector{Float64}}}(undef, length(processed))
for (i, (mz, y)) in enumerate(processed)
pk_mz, pk_int = detect_peaks_profile(mz, y; params_sym...)
peak_results[i] = (pk_mz, pk_int)
end
return peak_results
elseif step === :peaks_wavelet
peak_results = Vector{Tuple{Vector{Float64},Vector{Float64}}}(undef, length(processed))
for (i, (mz, y)) in enumerate(processed)
pk_mz, pk_int = detect_peaks_wavelet(mz, y; params_sym...)
peak_results[i] = (pk_mz, pk_int)
end
return peak_results
elseif step === :calibrate
return calibrate_spectra(processed; params_sym...)
else
@warn "Unsupported processing step: $step"
end
return processed
end
"""
run_versioned_preprocessing(session::MSISession, step::Symbol, params::Dict) -> VersionedSpectralData
Creates a new version of spectral data by applying a single processing step to the
current version in the session.
# Arguments
- `session::MSISession`: The current processing session.
- `step::Symbol`: The processing step to apply.
- `params::Dict`: Parameters for the processing step.
# Returns
- `VersionedSpectralData`: A new data version with the transformation applied.
"""
function run_versioned_preprocessing(session::MSISession, step::Symbol, params::Dict)
# Create a new version based on the current one
parent_version = session.current_version
new_version_id = string(uuid4())
# Apply the processing step
new_spectra = apply_processing_step(parent_version.spectra, step, params)
# Create the new versioned data object
new_version = VersionedSpectralData(
new_version_id,
parent_version.version_id,
new_spectra,
string(step),
now(),
params
)
return new_version
end
"""
apply_processing_with_version!(session::MSISession, step::Symbol, ui_params::Dict)
A high-level function to apply a processing step, create a new version, and
update the session state in place.
# Arguments
- `session::MSISession`: The session to modify.
- `step::Symbol`: The processing step to apply.
- `ui_params::Dict`: A dictionary of parameters, typically from a UI.
"""
function apply_processing_with_version!(session::MSISession, step::Symbol, ui_params::Dict)
# In a real app, you would validate and convert UI params here.
processing_params = ui_params
# Create the new version
new_version = run_versioned_preprocessing(session, step, processing_params)
# Update the session
push!(session.history, new_version)
session.current_version = new_version
# Describe the step for the history log
step_description = "$(string(step)) with params: " * join(["$k=$v" for (k,v) in processing_params], ", ")
push!(session.processing_steps, step_description)
return session
end
# =============================================================================
# Data Serialization & Export
# =============================================================================
"""
save_spectral_version(data::VersionedSpectralData, filepath::String)
Saves a `VersionedSpectralData` object to a file using the JLD2 format.
"""
function save_spectral_version(data::VersionedSpectralData, filepath::String)
JLD2.save_object(filepath, data)
end
"""
load_spectral_version(filepath::String) -> VersionedSpectralData
Loads a `VersionedSpectralData` object from a JLD2 file.
"""
function load_spectral_version(filepath::String)::VersionedSpectralData
return JLD2.load_object(filepath)
end
"""
_encode_base64(data::AbstractVector{<:Real}) -> String
Helper function to convert a numeric vector into a Base64 encoded string.
"""
function _encode_base64(data::AbstractVector{T}) where T <: Real
bytes = reinterpret(UInt8, data)
return base64encode(bytes)
end
"""
export_to_mzml(session::MSISession, filepath::String)
Exports the current version of the spectral data in a session to a standard
.mzML file.
# Arguments
- `session::MSISession`: The current session.
- `filepath::String`: The path for the output .mzML file.
"""
function export_to_mzml(session::MSISession, filepath::String)
spectra_to_export = session.current_version.spectra
open(filepath, "w") do f
# XML Header
write(f, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
write(f, "<mzML xmlns=\"http://psi.hupo.org/ms/mzml\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://psi.hupo.org/ms/mzml http://psidev.info/files/ms/mzML/xsd/mzML1.1.0.xsd\" version=\"1.1\">\n")
# CV List
write(f, " <cvList count=\"2\">\n")
write(f, " <cv id=\"MS\" fullName=\"Proteomics Standards Initiative Mass Spectrometry Ontology\" version=\"4.1.123\" URI=\"https://raw.githubusercontent.com/HUPO-PSI/psi-ms-CV/master/psi-ms.obo\"/>\n")
write(f, " <cv id=\"UO\" fullName=\"Unit Ontology\" version=\"07:03:2022\" URI=\"https://raw.githubusercontent.com/bio-ontology-research-group/unit-ontology/master/unit.obo\"/>\n")
write(f, " </cvList>\n")
# File Description
write(f, " <fileDescription>\n <fileContent>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000579\" name=\"MS1 spectrum\"/>\n")
write(f, " </fileContent>\n </fileDescription>\n")
# Run and Spectrum List
write(f, " <run id=\"run1\" defaultInstrumentConfigurationRef=\"instrument1\">\n")
write(f, " <spectrumList count=\"$(length(spectra_to_export))\" defaultDataProcessingRef=\"dp1\">\n")
for (i, (mz, intensity)) in enumerate(spectra_to_export)
# Ensure data is in the correct format for encoding
mz_64 = convert(Vector{Float64}, mz)
int_32 = convert(Vector{Float32}, intensity)
# Base64 encode the binary data
mz_b64 = _encode_base64(mz_64)
int_b64 = _encode_base64(int_32)
write(f, " <spectrum index=\"$(i-1)\" id=\"scan=$(i)\" defaultArrayLength=\"$(length(mz))\">\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000511\" name=\"ms level\" value=\"1\"/>\n")
# Assuming profile mode for processed data, could be made dynamic
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000128\" name=\"profile spectrum\"/>\n")
write(f, " <binaryDataArrayList count=\"2\">\n")
# m/z array
write(f, " <binaryDataArray encodedLength=\"$(length(mz_b64))\">\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000514\" name=\"m/z array\" unitCvRef=\"MS\" unitAccession=\"MS:1000040\" unitName=\"m/z\"/>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000523\" name=\"64-bit float\"/>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000576\" name=\"no compression\"/>\n")
write(f, " <binary>$(mz_b64)</binary>\n")
write(f, " </binaryDataArray>\n")
# Intensity array
write(f, " <binaryDataArray encodedLength=\"$(length(int_b64))\">\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000515\" name=\"intensity array\" unitCvRef=\"MS\" unitAccession=\"MS:1000131\" unitName=\"number of detector counts\"/>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000521\" name=\"32-bit float\"/>\n")
write(f, " <cvParam cvRef=\"MS\" accession=\"MS:1000576\" name=\"no compression\"/>\n")
write(f, " <binary>$(int_b64)</binary>\n")
write(f, " </binaryDataArray>\n")
write(f, " </binaryDataArrayList>\n")
write(f, " </spectrum>\n")
end
write(f, " </spectrumList>\n")
write(f, " </run>\n")
# Dummy instrument and data processing info
write(f, " <instrumentConfigurationList count=\"1\">\n")
write(f, " <instrumentConfiguration id=\"instrument1\"/>\n")
write(f, " </instrumentConfigurationList>\n")
write(f, " <dataProcessingList count=\"1\">\n")
write(f, " <dataProcessing id=\"dp1\"/>\n")
write(f, " </dataProcessingList>\n")
write(f, "</mzML>\n")
end
println("Successfully exported current data to $filepath")
end
# =============================================================================
# Stubs for UI Integration
# =============================================================================
"""
validate_ui_parameters(step::Symbol, ui_params::Dict) -> Tuple{Bool, String}
(Stub) Validates parameters from a UI before they are used in a processing step.
"""
function validate_ui_parameters(step::Symbol, ui_params::Dict)::Tuple{Bool, String}
# In a real implementation, this would check types, ranges, etc.
# For example, for :smooth, ensure 'sg_window' is an odd integer.
println("Validating parameters for step: $step")
return (true, "Parameters are valid.")
end
"""
run_processing_with_progress(session, step, params, progress_callback)
(Stub) A wrapper for running a processing step that includes a progress reporting callback.
"""
function run_processing_with_progress(session, step, params, progress_callback)
progress_callback(0.0, "Starting $step...")
# This is a simplified example. Real implementation would need to
# hook into the loops inside apply_processing_step.
new_version = run_versioned_preprocessing(session, step, params)
progress_callback(1.0, "Finished $step.")
return new_version
end
"""
safe_processing_application!(session::MSISession, step::Symbol, params::Dict)
(Stub) A safe wrapper to apply a processing step that includes error handling and rollback.
"""
function safe_processing_application!(session::MSISession, step::Symbol, params::Dict)
num_history = length(session.history)
try
println("Safely applying step: $step")
apply_processing_with_version!(session, step, params)
catch e
@error "Processing step '$step' failed!" exception=(e, catch_backtrace())
# Roll back to the previous state
while length(session.history) > num_history
undo_last_step!(session)
end
println("Session has been rolled back to the previous state.")
return session # Return the rolled-back session
end
return session # Return the updated session
end

4
test/readme.md
View File

@ -40,10 +40,10 @@ Before running the tests, you must edit the `test/run_tests.jl` file to point to
2. **Execute the Different Test Scripts**:
Run the following command from the project's root directory. This will install the necessary dependencies and run the tests.
```bash
julia --project=. test/run_tests.jl
julia --threads auto --project=. test/run_tests.jl
```
```bash
julia --project=. test/run_preprocessing.jl
julia --threads auto --project=. test/run_preprocessing.jl
```
3. **Check the Results**:

735
test/run_preprocessing.jl
View File

@ -1,468 +1,371 @@
# test/run_preprocessing.jl
# ===================================================================
# Test Environment for the Preprocessing.jl Module
# Preprocessing Test Suite for JuliaMSI
# ===================================================================
# This script tests the full preprocessing pipeline on single spectra
# and total spectra from both .mzML and .imzML files.
# It generates an overlay plot showing all preprocessing stages and
# saves the resulting feature matrix to a CSV file.
# This script provides a customizable workflow to test and visualize
# the effects of different preprocessing steps on individual spectra
# from .mzML or .imzML files.
#
# Instructions:
# 1. Ensure the file paths in the "CONFIG" section are correct.
# 1. Configure the file paths and parameters in the "CONFIG" section.
# 2. Run the script from the project's root directory:
# julia test/run_preprocessing.jl
# 3. Check the `test/results/` folder for output plots and CSVs.
# julia --threads auto --project=. test/run_preprocessing.jl
# 3. Check the configured `RESULTS_DIR` for output plots and CSV files.
# ===================================================================
using Printf
using CairoMakie
using DataFrames
using CSV
using Statistics
using Interpolations
import Pkg
using DataFrames # For saving FeatureMatrix to CSV
using CSV # For saving FeatureMatrix to CSV
using Statistics # For mean()
# --- Load Modules ---
# Activate the project environment to access dependencies
# --- Load the MSI_src Module ---
# Activate the project environment at the parent directory of this test script
Pkg.activate(joinpath(@__DIR__, ".."))
using MSI_src # This brings in Preprocessing.jl functions via export
using MSI_src
# ===================================================================
# CONFIG: Test files and parameters
# CONFIG: CUSTOMIZE YOUR PREPROCESSING WORKFLOW HERE
# ===================================================================
# --- Test Files ---
# An mzML file for testing spectrum-based processing
# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML"
# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/set de datos MS/Leaf_profile_LD_LTP_MS.mzML"
const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/set de datos MS/Escopolamina_tuneo_fraq_20ev.mzML"
#const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/set de datos MS/Atropina_tuneo_fraq_20ev.mzML"
# --- Input and Output ---
const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML" # Can be .mzML or .imzML
const RESULTS_DIR = "test/results/preprocessing"
const NUM_SPECTRA_TO_PROCESS = nothing # Set to `nothing` to process all spectra
const MZML_SPECTRUM_ID = 1
# --- Internal Standards for Calibration and QC ---
# replace these with m/z values of known compounds present in your dataset.
const INTERNAL_STANDARDS = Dict(
"P13" => 432.6584,
"P15" => 464.6059,
"P17" => 526.5534,
"P21" => 650.4485,
"P25" => 774.3435,
"P29" => 898.2385,
"P31" => 950.1861,
"P33" => 1022.1336,
"P37" => 1146.0286,
"P45" => 1593.8187,
"Unknown 1" => 772.433,
"Unknown 2" => 772.5253
)
# An imzML file for testing
# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.imzML"
# const IMZML_COORDS = (50, 50)
const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML"
const IMZML_COORDS = (1997, 639)
# --- Preprocessing Step Parameters ---
# --- Output Directory ---
const RESULTS_DIR = "test/results"
# Step 0: Quality Control (QC)
const QC_PARAMS = (
ppm_tolerance = 5.0, # PPM tolerance for matching internal standards
)
# Step 1: Calibration
const CALIBRATION_PARAMS = (
enabled = true,
ppm_tolerance = 5.0, # PPM tolerance for finding calibration peaks
)
# Step 2: Smoothing
# Note: Smoothing is less effective and often unnecessary for centroid data.
const SMOOTHING_PARAMS = (
enabled = false,
window = 9,
order = 2,
)
# Step 3: Baseline Correction
# Note: Baseline correction is less effective and often unnecessary for centroid data.
const BASELINE_CORRECTION_PARAMS = (
enabled = false,
iterations = 100,
)
# Step 4: Normalization
const NORMALIZATION_PARAMS = (
enabled = true,
method = :tic, # :tic, :median, or :none
)
# Step 5: Peak Detection
const PEAK_DETECTION_PARAMS = (
enabled = true,
method = :centroid, # Use :centroid for centroided data, :profile for profile data
# --- Parameters for :centroid method ---
intensity_threshold = 0.0, # Filters out peaks below this absolute intensity
# --- Parameters for :profile method ---
half_window = 10,
snr_threshold = 3.0,
min_peak_prominence = 0.05,
merge_peaks_tolerance = 0.002,
)
# Step 6: Peak Alignment (Warping)
# This is performed after collecting peaks from all spectra.
const PEAK_ALIGNMENT_PARAMS = (
enabled = true,
tolerance = 0.002,
tolerance_unit = :mz, # :mz or :ppm
min_matched_peaks = 3, # Lowered for sparse centroid data
)
# Step 7: Peak Binning
const PEAK_BINNING_PARAMS = (
enabled = true,
tolerance = 0.1,
tolerance_unit = :mz, # :mz or :ppm
frequency_threshold = 0.1, # Min fraction of spectra a peak must be in
)
# ===================================================================
# HELPER FUNCTIONS FOR PLOTTING
# HELPER FUNCTIONS
# ===================================================================
"""
plot_overlay_stages(collected_data, output_path, title)
plot_spectrum_on_fig(fig_pos, mz, intensity, title)
Creates a single plot overlaying spectra from different preprocessing stages.
Helper function to plot a spectrum on a specific position of a CairoMakie figure.
"""
function plot_overlay_stages(collected_data, output_path, title)
fig = Figure(size = (1400, 800))
ax = Axis(fig[1, 1], title=title, xlabel="m/z", ylabel="Intensity")
colors = Makie.wong_colors() # A good set of distinct colors
for (i, (stage, mz, intensity)) in enumerate(collected_data)
color = colors[mod1(i, length(colors))] # Cycle through colors
# Plot the spectrum as a line
lines!(ax, mz, intensity, color=color, label=string(stage))
# If it's the peaks stage, also mark the peak tops
if stage == :peaks
scatter!(ax, mz, intensity, color=color, marker=:circle, markersize=8, label="$(string(stage)) (tops)")
end
end
axislegend(ax, position=:rt) # Right top position
save(output_path, fig)
println("SUCCESS: Overlay plot saved to $output_path")
function plot_spectrum_on_fig(fig_pos, mz, intensity, title)
ax = Axis(fig_pos, title=title, xlabel="m/z", ylabel="Intensity")
lines!(ax, mz, intensity)
end
# ===================================================================
# TEST DEFINITIONS
# MAIN PROCESSING SCRIPT
# ===================================================================
"""
test_full_pipeline(msi_data, spectrum_id; output_dir, file_type_prefix)
function run_preprocessing_suite()
println("Starting Preprocessing Test Suite...")
mkpath(RESULTS_DIR)
Tests the full preprocessing pipeline on a single spectrum and saves a plot
for each intermediate step using the `on_stage` callback.
`spectrum_id` can be an `Int` (for mzML) or a `Tuple{Int, Int}` (for imzML).
"""
function test_full_pipeline(msi_data, spectrum_id; output_dir, file_type_prefix, mz_tolerance=0.002)
println("\n--- Testing Full Preprocessing Pipeline on Spectrum: $spectrum_id (File Type: $file_type_prefix) ---")
# --- Load Data ---
println("Loading data from: $TEST_FILE")
if !isfile(TEST_FILE)
println("ERROR: Test file not found. Please check the path in the CONFIG section.")
return
end
msi_data = @time OpenMSIData(TEST_FILE)
println("Data loaded successfully. Found $(length(msi_data.spectra_metadata)) spectra.")
# 1. Determine the spectrum index
local spec_idx
if spectrum_id isa Int
spec_idx = spectrum_id
else # Tuple for imzML
spec_idx = msi_data.coordinate_map[spectrum_id...]
# --- Determine which spectra to process ---
total_spectra = length(msi_data.spectra_metadata)
indices_to_process = if NUM_SPECTRA_TO_PROCESS === nothing
1:total_spectra
else
unique(round.(Int, range(1, total_spectra, length=min(NUM_SPECTRA_TO_PROCESS, total_spectra))))
end
println("Will process $(length(indices_to_process)) spectra.")
# --- Data storage for results ---
qc_results = DataFrame(spectrum_idx=Int[], metric=String[], value=Float64[], compound=String[])
all_processed_peaks_mz = Vector{Vector{Float64}}()
all_processed_peaks_int = Vector{Vector{Float64}}()
spectrum_indices_with_peaks = Int[]
# --- Main Loop: Process each spectrum individually ---
println("\n" * "="^20 * " Processing Individual Spectra " * "="^20)
for (i, idx) in enumerate(indices_to_process)
print("\rProcessing spectrum #$idx ($(i)/$(length(indices_to_process)))...")
process_spectrum(msi_data, idx) do mz, intensity
if qc_is_empty(mz, intensity) || !qc_is_regular(mz)
# @warn "Skipping empty or irregular spectrum #$idx"
return
end
original_mz, original_intensity = copy(mz), copy(intensity)
processed_mz, processed_intensity = copy(mz), copy(intensity)
# --- Step 0: QC (PPM and Resolution) ---
# This happens before any modification to the m/z axis
let ref_masses = collect(values(INTERNAL_STANDARDS))
matched_peaks = find_calibration_peaks(processed_mz, processed_intensity, ref_masses, ppm_tolerance=QC_PARAMS.ppm_tolerance)
for (compound, theoretical_mz) in INTERNAL_STANDARDS
# Find the measured m/z that corresponds to this theoretical_mz
measured_mz = 0.0
for (theo, meas) in matched_peaks
if theo == theoretical_mz
measured_mz = meas
break
end
end
if measured_mz > 0
# PPM Error
ppm_error = calculate_ppm_error(measured_mz, theoretical_mz)
push!(qc_results, (idx, "ppm_error", ppm_error, compound))
# Resolution
resolution = calculate_resolution_fwhm(measured_mz, processed_mz, processed_intensity)
if !isnan(resolution)
push!(qc_results, (idx, "resolution", resolution, compound))
end
end
end
end
# --- Step 1: Calibration ---
if CALIBRATION_PARAMS.enabled
ref_masses = collect(values(INTERNAL_STANDARDS))
matched_peaks = find_calibration_peaks(processed_mz, processed_intensity, ref_masses, ppm_tolerance=CALIBRATION_PARAMS.ppm_tolerance)
if length(matched_peaks) >= 2
measured = sort(collect(values(matched_peaks)))
theoretical = sort(collect(keys(matched_peaks)))
itp = linear_interpolation(measured, theoretical, extrapolation_bc=Line())
processed_mz = itp(processed_mz)
end
end
# --- Step 2: Smoothing ---
if SMOOTHING_PARAMS.enabled
processed_intensity = smooth_spectrum(processed_intensity, window=SMOOTHING_PARAMS.window, order=SMOOTHING_PARAMS.order)
end
# --- Step 3: Baseline Correction ---
if BASELINE_CORRECTION_PARAMS.enabled
baseline = snip_baseline(processed_intensity, iterations=BASELINE_CORRECTION_PARAMS.iterations)
processed_intensity .-= baseline
processed_intensity = max.(0, processed_intensity) # Ensure non-negativity
end
# --- Step 4: Normalization ---
if NORMALIZATION_PARAMS.enabled && NORMALIZATION_PARAMS.method != :none
if NORMALIZATION_PARAMS.method == :tic
processed_intensity = tic_normalize(processed_intensity)
elseif NORMALIZATION_PARAMS.method == :median
processed_intensity = median_normalize(processed_intensity)
end
end
# --- Step 5: Peak Detection ---
local pk_mz, pk_int
pk_mz, pk_int = Float64[], Float64[] # Initialize empty
if PEAK_DETECTION_PARAMS.enabled
if PEAK_DETECTION_PARAMS.method == :profile
pk_mz, pk_int = detect_peaks_profile(processed_mz, processed_intensity,
half_window=PEAK_DETECTION_PARAMS.half_window,
snr_threshold=PEAK_DETECTION_PARAMS.snr_threshold,
min_peak_prominence=PEAK_DETECTION_PARAMS.min_peak_prominence,
merge_peaks_tolerance=PEAK_DETECTION_PARAMS.merge_peaks_tolerance)
elseif PEAK_DETECTION_PARAMS.method == :wavelet
pk_mz, pk_int = detect_peaks_wavelet(processed_mz, processed_intensity)
else # :centroid
pk_mz, pk_int = detect_peaks_centroid(processed_mz, processed_intensity)
end
if !isempty(pk_mz)
push!(all_processed_peaks_mz, pk_mz)
push!(all_processed_peaks_int, pk_int)
push!(spectrum_indices_with_peaks, idx)
end
end
# --- Visualization of a sample spectrum ---
if i == 1 # Only plot the first processed spectrum
println("\nGenerating example plots for spectrum #$idx...")
fig = Figure(size=(1200, 800))
plot_spectrum_on_fig(fig[1,1], original_mz, original_intensity, "1. Original Spectrum")
plot_spectrum_on_fig(fig[2,1], processed_mz, processed_intensity, "2. After All Steps (Before Peak Picking)")
# Plot detected peaks
ax = Axis(fig[3,1], title="3. Detected Peaks")
if !isempty(pk_mz)
stem!(ax, pk_mz, pk_int)
end
save(joinpath(RESULTS_DIR, "example_spectrum_processing.png"), fig)
println("Saved example processing plots.")
end
end
end
println("\nIndividual spectrum processing complete.")
# --- Save QC Results ---
if !isempty(qc_results)
println("\n" * "="^20 * " Generating QC Report " * "="^20)
CSV.write(joinpath(RESULTS_DIR, "qc_results.csv"), qc_results)
println("QC results saved to qc_results.csv")
# Generate summary plots for QC
fig = Figure(size=(1200, 600))
# PPM Error Histogram
ppm_errors = filter(row -> row.metric == "ppm_error", qc_results).value
if !isempty(ppm_errors)
ax1 = Axis(fig[1,1], title="PPM Error Distribution", xlabel="PPM Error")
hist!(ax1, ppm_errors, bins=30)
end
# Resolution Histogram
resolutions = filter(row -> row.metric == "resolution", qc_results).value
if !isempty(resolutions)
ax2 = Axis(fig[1,2], title="Resolution Distribution", xlabel="Resolution (FWHM)")
hist!(ax2, resolutions, bins=30)
end
save(joinpath(RESULTS_DIR, "qc_summary_plots.png"), fig)
println("QC summary plots saved.")
end
if spec_idx == 0
println("SKIPPED: No spectrum found at coordinates $spectrum_id.")
if isempty(all_processed_peaks_mz)
@warn "No peaks were detected in any of the processed spectra. Skipping alignment and binning."
return
end
# 2. Define the pipeline steps in the desired order
pipeline_steps = [
:qc,
:transform,
:smooth,
:baseline,
:normalize,
:peaks,
:align, # Align requires multiple spectra, but we'll run it on a single one for now (will warn)
:bin
]
# --- Step 6: Peak Alignment ---
println("\n" * "="^20 * " Aligning Peaks " * "="^20)
aligned_peaks_mz = copy(all_processed_peaks_mz)
if PEAK_ALIGNMENT_PARAMS.enabled
# Create a reference peak list (e.g., from the spectrum with the most peaks)
ref_idx = argmax(length.(all_processed_peaks_mz))
reference_peaks = all_processed_peaks_mz[ref_idx]
println("Using spectrum $(spectrum_indices_with_peaks[ref_idx]) as alignment reference.")
# Define parameters for each step
params = Dict(
:transform_method => :sqrt,
:sg_window => 15,
:sg_order => 2,
:snip_iterations => 100,
:normalize_method => :tic,
:peak_half_window => 10,
:peak_snr => 3.0,
:peak_intensity_threshold => 0.0, # For centroid peak detection
:align_tolerance => mz_tolerance,
:bin_tolerance => mz_tolerance,
:bin_min_frequency => 0.0 # Keep all bins for a single spectrum
)
for i in 1:length(aligned_peaks_mz)
if i == ref_idx continue end
alignment_func = align_peaks_lowess(reference_peaks, aligned_peaks_mz[i],
tolerance=PEAK_ALIGNMENT_PARAMS.tolerance,
tolerance_unit=PEAK_ALIGNMENT_PARAMS.tolerance_unit,
min_matched_peaks=PEAK_ALIGNMENT_PARAMS.min_matched_peaks)
aligned_peaks_mz[i] = alignment_func(aligned_peaks_mz[i])
end
println("Peak alignment complete.")
end
# 3. Define the on_stage callback to collect data for overlay plot and save separate plots
collected_stage_data = []
stage_counter = Ref(0) # Initialize counter for sequential naming
normalized_spectrum = nothing # Variable to hold the normalized spectrum
function stage_callback(stage; idx, mz, intensity)
stage_counter[] += 1 # Increment counter
println(" -> Generating plot for stage: $stage")
# --- Step 7: Peak Binning & Feature Matrix Generation ---
println("\n" * "="^20 * " Binning Peaks " * "="^20)
if PEAK_BINNING_PARAMS.enabled
feature_matrix, mz_bins = bin_peaks(aligned_peaks_mz, all_processed_peaks_int,
PEAK_BINNING_PARAMS.tolerance,
tolerance_unit=PEAK_BINNING_PARAMS.tolerance_unit,
frequency_threshold=PEAK_BINNING_PARAMS.frequency_threshold)
local fig # Make fig available in the whole function scope
if stage == :normalize
normalized_spectrum = (mz, intensity)
fig = plot_stage_spectrum(mz, intensity, title="Stage: $stage (Spectrum $spectrum_id)")
elseif stage == :peaks && normalized_spectrum !== nothing
# For the peaks stage, plot the normalized spectrum as a base layer
fig = Figure(size = (1400, 500))
ax = Axis(fig[1, 1], title="Stage: Peaks (Spectrum $spectrum_id)", xlabel="m/z", ylabel="Intensity")
lines!(ax, normalized_spectrum[1], normalized_spectrum[2], color=:gray, label="Normalized Spectrum")
scatter!(ax, mz, intensity, color=:red, marker=:circle, markersize=8, label="Detected Peaks")
axislegend(ax)
if !isempty(feature_matrix)
df = DataFrame(feature_matrix, :auto)
rename!(df, ["bin_$(i)" for i in 1:size(df, 2)])
insertcols!(df, 1, :spectrum_idx => spectrum_indices_with_peaks)
CSV.write(joinpath(RESULTS_DIR, "feature_matrix.csv"), df)
println("Feature matrix saved to feature_matrix.csv")
# Save bin m/z ranges
bin_df = DataFrame(bin_index=1:length(mz_bins), mz_start=[b[1] for b in mz_bins], mz_end=[b[2] for b in mz_bins])
CSV.write(joinpath(RESULTS_DIR, "bin_definitions.csv"), bin_df)
println("Bin definitions saved to bin_definitions.csv")
else
# Default plotting for all other stages
fig = plot_stage_spectrum(mz, intensity, title="Stage: $stage (Spectrum $spectrum_id)")
@warn "Feature matrix was empty after binning."
end
# Save the figure
stage_output_path = joinpath(output_dir, "$(file_type_prefix)_$(spectrum_id)_$(stage_counter[])_$(stage).png")
save(stage_output_path, fig)
# Collect data for overlay plot
push!(collected_stage_data, (stage, mz, intensity))
end
# 4. Run the pipeline on the single spectrum
println("Running pipeline with steps: $pipeline_steps")
processed_result = run_preprocessing_pipeline(
msi_data,
[spec_idx], # The pipeline expects a vector of indices
steps=pipeline_steps,
params=params,
on_stage=stage_callback
)
# 5. Generate and save the overlay plot
overlay_output_path = joinpath(output_dir, "$(file_type_prefix)_$(spectrum_id)_all_stages_overlay.png")
plot_overlay_stages(collected_stage_data, overlay_output_path, "Preprocessing Stages Overlay (Spectrum $spectrum_id)")
# 6. Save feature matrix if generated
if processed_result isa FeatureMatrix
feature_matrix_output_path = joinpath(output_dir, "$(file_type_prefix)_$(spectrum_id)_feature_matrix.csv")
# Convert mz_bins to a more readable format for CSV
mz_labels = ["$(round(b[1], digits=4))_$(round(b[2], digits=4))" for b in processed_result.mz_bins]
df = DataFrame(processed_result.matrix, Symbol.(mz_labels))
CSV.write(feature_matrix_output_path, df)
println("SUCCESS: Feature matrix saved to $feature_matrix_output_path")
else
@warn "Pipeline did not return a FeatureMatrix for Spectrum $spectrum_id."
processed_result
end
println("--- Pipeline test finished for Spectrum: $spectrum_id (File Type: $file_type_prefix) ---")
println("Check the '$(output_dir)' directory for output plots and CSVs.")
println("\nPreprocessing Test Suite finished successfully!")
end
"""
test_full_pipeline_on_total_spectrum(msi_data; output_dir, file_type_prefix)
Tests the full preprocessing pipeline on the *total spectrum* (sum of all spectra)
and saves a plot for each intermediate step.
"""
function test_full_pipeline_on_total_spectrum(msi_data; output_dir, file_type_prefix, mz_tolerance=0.002)
println("\n--- Testing Full Preprocessing Pipeline on TOTAL Spectrum (File Type: $file_type_prefix) ---")
# 1. Get the total spectrum
total_mz, total_intensity = get_total_spectrum(msi_data)
total_spectrum = (total_mz, total_intensity)
if qc_is_empty(total_mz, total_intensity)
println("SKIPPED: Total spectrum is empty.")
return
end
# 2. Define the pipeline steps and parameters (same as for single spectrum)
pipeline_steps = [
:qc,
:transform,
:smooth,
:baseline,
:normalize,
:peaks,
:align, # Align requires multiple spectra, but we'll run it on a single one for now (will warn)
:bin
]
params = Dict(
:transform_method => :sqrt,
:sg_window => 15,
:sg_order => 2,
:snip_iterations => 100,
:normalize_method => :tic,
:peak_half_window => 10,
:peak_snr => 3.0,
:align_tolerance => mz_tolerance,
:bin_tolerance => mz_tolerance,
:bin_min_frequency => 0.0 # Keep all bins for a single spectrum
)
# 3. Define the on_stage callback
collected_stage_data = []
stage_counter = Ref(0) # Initialize counter for sequential naming
normalized_spectrum_total = nothing # Variable to hold the normalized spectrum
function stage_callback_total(stage; idx, mz, intensity)
stage_counter[] += 1 # Increment counter
println(" -> Generating plot for stage: $stage (Total Spectrum)")
local fig
if stage == :normalize
normalized_spectrum_total = (mz, intensity)
fig = plot_stage_spectrum(mz, intensity, title="Stage: $stage (Total Spectrum)")
elseif stage == :peaks && normalized_spectrum_total !== nothing
fig = Figure(size = (1400, 500))
ax = Axis(fig[1, 1], title="Stage: Peaks (Total Spectrum)", xlabel="m/z", ylabel="Intensity")
lines!(ax, normalized_spectrum_total[1], normalized_spectrum_total[2], color=:gray, label="Normalized Spectrum")
scatter!(ax, mz, intensity, color=:red, marker=:circle, markersize=8, label="Detected Peaks")
axislegend(ax)
else
fig = plot_stage_spectrum(mz, intensity, title="Stage: $stage (Total Spectrum)")
end
# Save separate plot with sequential name
stage_output_path = joinpath(output_dir, "$(file_type_prefix)_total_$(stage_counter[])_$(stage).png")
save(stage_output_path, fig)
# Collect data for overlay plot
push!(collected_stage_data, (stage, mz, intensity))
end
# 4. Run the pipeline on the single total spectrum
println("Running pipeline with steps: $pipeline_steps")
processed_result = run_preprocessing_pipeline(
[total_spectrum], # Pass the total spectrum as a vector of one spectrum
steps=pipeline_steps,
params=params,
on_stage=stage_callback_total
)
# 5. Generate and save the overlay plot
overlay_output_path = joinpath(output_dir, "$(file_type_prefix)_total_all_stages_overlay.png")
plot_overlay_stages(collected_stage_data, overlay_output_path, "Preprocessing Stages Overlay (Total Spectrum)")
# 6. Save feature matrix if generated
if processed_result isa FeatureMatrix
feature_matrix_output_path = joinpath(output_dir, "$(file_type_prefix)_total_feature_matrix.csv")
# Convert mz_bins to a more readable format for CSV
mz_labels = ["$(round(b[1], digits=4))_$(round(b[2], digits=4))" for b in processed_result.mz_bins]
df = DataFrame(processed_result.matrix, Symbol.(mz_labels))
CSV.write(feature_matrix_output_path, df)
println("SUCCESS: Feature matrix saved to $feature_matrix_output_path")
else
@warn "Pipeline did not return a FeatureMatrix for Total Spectrum."
processed_result
end
println("--- Pipeline test finished for TOTAL Spectrum (File Type: $file_type_prefix) ---")
println("Check the '$(output_dir)' directory for output plots and CSVs.")
end
# ===================================================================
# TEST RUNNER
# ===================================================================
function run_preprocessing_tests()
println("="^80)
println("STARTING PREPROCESSING TEST SUITE")
println("="^80)
# --- Test Case 1: Run full pipeline on a single mzML spectrum ---
println("\n" * "="^20 * " Test Case 1: Full Pipeline on .mzML Spectrum " * "="^20)
println("FILE: ", TEST_MZML_FILE)
if isfile(TEST_MZML_FILE)
try
msi_data_mzml = OpenMSIData(TEST_MZML_FILE)
# Dynamically determine tolerance
println("\n--- Calculating optimal tolerance for .mzML data ---")
report_mzml = analyze_mass_accuracy(msi_data_mzml, get_common_calibration_standards(:maldi_pos))
mz_tolerance_mzml = 0.002 # Default
if haskey(report_mzml, :optimal_ppm) && !isnan(report_mzml.optimal_ppm) && !isempty(report_mzml.matched_peaks)
avg_mz = mean([p[1] for p in report_mzml.matched_peaks])
mz_tolerance_mzml = avg_mz * report_mzml.optimal_ppm / 1e6
println("Optimal PPM: $(round(report_mzml.optimal_ppm, digits=2)), Average m/z: $(round(avg_mz, digits=2))")
println("Calculated m/z tolerance: $(round(mz_tolerance_mzml, digits=5))")
else
println("Could not determine optimal tolerance, using default: $mz_tolerance_mzml")
end
# Create a dedicated subdirectory for the output plots
mzml_output_dir = joinpath(RESULTS_DIR, "mzml_pipeline_stages")
mkpath(mzml_output_dir)
test_full_pipeline(msi_data_mzml, MZML_SPECTRUM_ID, output_dir=mzml_output_dir, file_type_prefix="mzml", mz_tolerance=mz_tolerance_mzml)
test_full_pipeline_on_total_spectrum(msi_data_mzml, output_dir=mzml_output_dir, file_type_prefix="mzml", mz_tolerance=mz_tolerance_mzml)
catch e
println("ERROR in .mzML pipeline test: $e")
showerror(stdout, e, catch_backtrace())
end
else
println("SKIPPED: File not found: $TEST_MZML_FILE")
end
# --- Test Case 2: Run full pipeline on a single imzML spectrum ---
println("\n" * "="^20 * " Test Case 2: Full Pipeline on .imzML Spectrum " * "="^20)
println("FILE: ", TEST_IMZML_FILE)
if isfile(TEST_IMZML_FILE)
try
msi_data_imzml = OpenMSIData(TEST_IMZML_FILE)
# Dynamically determine tolerance
println("\n--- Calculating optimal tolerance for .imzML data ---")
report_imzml = analyze_mass_accuracy(msi_data_imzml, get_common_calibration_standards(:maldi_pos))
mz_tolerance_imzml = 0.002 # Default
if haskey(report_imzml, :optimal_ppm) && !isnan(report_imzml.optimal_ppm) && !isempty(report_imzml.matched_peaks)
avg_mz = mean([p[1] for p in report_imzml.matched_peaks])
mz_tolerance_imzml = avg_mz * report_imzml.optimal_ppm / 1e6
println("Optimal PPM: $(round(report_imzml.optimal_ppm, digits=2)), Average m/z: $(round(avg_mz, digits=2))")
println("Calculated m/z tolerance: $(round(mz_tolerance_imzml, digits=5))")
else
println("Could not determine optimal tolerance, using default: $mz_tolerance_imzml")
end
# Create a dedicated subdirectory for the output plots
imzml_output_dir = joinpath(RESULTS_DIR, "imzml_pipeline_stages")
mkpath(imzml_output_dir)
test_full_pipeline(msi_data_imzml, IMZML_COORDS, output_dir=imzml_output_dir, file_type_prefix="imzml", mz_tolerance=mz_tolerance_imzml)
test_full_pipeline_on_total_spectrum(msi_data_imzml, output_dir=imzml_output_dir, file_type_prefix="imzml", mz_tolerance=mz_tolerance_imzml)
# generate_qc_report(msi_data_imzml, TEST_IMZML_FILE, output_dir=imzml_output_dir)
custom_reference_peaks = Dict(
31.974 => "Red Phosphorus",
432.6584 => "P13",
464.6059 => "P15",
526.5534 => "P17",
650.4485 => "P21",
774.3435 => "P25",
898.2385 => "P29",
950.1861 => "P31",
1022.1336 => "P33",
1146.0286 => "P37",
1593.8187 => "P45",
772.433 => "Unknown 1",
772.5253 => "Unknown 2"
)
n_samples = length(msi_data_imzml.spectra_metadata)
generate_qc_report(msi_data_imzml, TEST_IMZML_FILE, reference_peaks=custom_reference_peaks, output_dir=imzml_output_dir, sample_spectra=n_samples)
catch e
println("ERROR in .imzML pipeline test: $e")
showerror(stdout, e, catch_backtrace())
end
else
println("SKIPPED: File not found: $TEST_IMZML_FILE")
end
# --- Test Case 3: Generate QC Report for .imzML data ---
println("\n" * "="^20 * " Test Case 3: QC Report Generation for .imzML " * "="^20)
println("FILE: ", TEST_IMZML_FILE)
if isfile(TEST_IMZML_FILE)
try
msi_data_imzml = OpenMSIData(TEST_IMZML_FILE)
# Create a dedicated subdirectory for the QC report
qc_output_dir = joinpath(RESULTS_DIR, "qc_report")
mkpath(qc_output_dir)
println("\n--- Generating comprehensive QC report ---")
custom_reference_peaks = Dict(
31.974 => "Red Phosphorus",
432.6584 => "P13",
464.6059 => "P15",
526.5534 => "P17",
650.4485 => "P21",
774.3435 => "P25",
898.2385 => "P29",
950.1861 => "P31",
1022.1336 => "P33",
1146.0286 => "P37",
1593.8187 => "P45",
772.433 => "Unknown 1",
772.5253 => "Unknown 2"
)
# You can control the number of spectra sampled for the QC report.
# For the most accurate results, you can sample all spectra, but it will take longer.
# To sample all, use: n_samples = length(msi_data_imzml.spectra_metadata)
n_samples = length(msi_data_imzml.spectra_metadata)
#generate_qc_report(msi_data_imzml, TEST_IMZML_FILE, output_dir=qc_output_dir)
generate_qc_report(msi_data_imzml, TEST_IMZML_FILE, reference_peaks=custom_reference_peaks, output_dir=qc_output_dir, sample_spectra=n_samples)
println("\n--- Analyzing specific reference peaks ---")
#=
reference_peaks = Dict(
104.10754 => "Imidazole",
175.11995 => "GPC fragment",
226.15687 => "Phosphocholine"
)
=#
report = analyze_mass_accuracy(msi_data_imzml, custom_reference_peaks)
if haskey(report, :optimal_ppm)
println("Optimal PPM tolerance with specific peaks: $(round(report.optimal_ppm, digits=2)) ppm")
else
println("Could not determine optimal PPM with specific peaks.")
end
catch e
println("ERROR in QC report generation test: $e")
showerror(stdout, e, catch_backtrace())
end
else
println("SKIPPED: File not found: $TEST_IMZML_FILE")
end
println("\nPreprocessing tests finished.")
end
# --- Execute ---
# Ensure the results directory exists
mkpath(RESULTS_DIR)
@time run_preprocessing_tests()
run_preprocessing_suite()