Julian/JuliaMSI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dev/Memory_pooling_adjustments #1

Merged
Julian merged 6 commits from dev/Memory_pooling_adjustments into main 2026-05-31 01:03:57 +02:00
Conversation 0 Commits 6 Files Changed 31 +262 -13
7 changed files with 262 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 70fff16170 - Show all commits

4
.gitignore vendored
View File

@ -7,4 +7,6 @@ log/*
R_original_scripts/
test/results/
test/binarization_results/
Manifest.toml
Manifest.toml
MSI_sysimage.dll
MSI_sysimage.so

5
Project.toml
View File

@ -35,6 +35,7 @@ Loess = "4345ca2d-374a-55d4-8d30-97f9976e7612"
Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4"
NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85"
PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d"
Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a"
PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
@ -52,9 +53,9 @@ Accessors = "0.1"
Base64 = "1.11"
CSV = "0.10"
CairoMakie = "0.13"
CodecBase = "0.3"
ColorSchemes = "3.30"
Colors = "0.12"
CodecBase = "0.3"
ContinuousWavelets = "1.1"
DataFrames = "1.7"
Dates = "1.11"
@ -63,7 +64,6 @@ GLMakie = "0.11"
Genie = "5.31"
GenieFramework = "2.8"
HistogramThresholding = "0.3"
Interpolations = "0.15"
ImageBinarization = "0.3"
ImageComponentAnalysis = "0.2"
ImageContrastAdjustment = "0.3"
@ -72,6 +72,7 @@ ImageFiltering = "0.7"
ImageMorphology = "0.4"
ImageSegmentation = "1.9"
Images = "0.26"
Interpolations = "0.15"
JLD2 = "0.6"
JSON = "0.21"
Libz = "1.0"

34
README.md
View File

@ -22,7 +22,7 @@ https://codeberg.org/LabABI/JuliaMSI
Example of a correct route:<br>
~/Downloads/JuliaMSI-main/juliamsi
2. Without entering the Julia environment, launch the project in your terminal with the following command (which works for all operating systems):
```
```bash
julia --threads auto --project=. start_MSI_GUI.jl
```
3. After the script has finished loading, you can open a [page](http://127.0.0.1:1481/) in your browser with the web app running.
@ -34,6 +34,38 @@ Minimum system requirements: 4 core processor, 8 GB RAM<br>
JuliaMSI is a Graphical User Interface for a library of MSI tools in Julia: https://github.com/CINVESTAV-LABI/julia_mzML_imzML
## Build the System Image (One-time)
Alternatively, you can generate the .so file by running the next build script in your directory:
```bash
julia --project=. build_sysimage.jl
```
This may take 5–10 minutes as it merges all dependencies and JuliaMSI functions into a single binary. The resulting .so/.dll file will be around **300MB–600MB** because it contains the pre-compiled machine code for your entire environment.
A sysimage built on Linux (.so) will not work on Windows.
You must run the build_sysimage.jl script once on each target operating system.
Once MSI_sysimage.so is created in your directory, adapt your command like this:
```bash
julia --project=. -e 'using Pkg; Pkg.precompile()'
# This will find MSI_sysimage.so, .dll, or .dylib automatically:
julia --threads auto --project=. --sysimage MSI_sysimage* start_MSI_GUI.jl
```
The command above loads a pre-compiled version of your environment (note, you have to use either the sysimage command or the normal start command), speeding up the booting delay. Your GUI or scripts using JuliaMSI should start almost instantly and process files significantly faster than before.
Always run the build script **in the same project root** where you intend to run the app.
In scripts like test/run_tests.jl, you no longer need to manually include("../src/MSI_src.jl"). You can simply treat it like a globally installed package to load the precompiled binary:
```julia
using MSI_src
# ... rest of your script
```
### Run with the --sysimage flag
Launch your tests using the same flag to bypass all compilation overhead:
```bash
julia --threads auto --project=. --sysimage MSI_sysimage.so test/run_tests.jl
```
## License
JuliaMSI is published under the terms of the MIT License.

14
app.jl
View File

@ -2422,8 +2422,22 @@ end
else
push!(newly_created_folders, replace(basename(file_path), r"\.imzML$"i => ""))
end
# --- Periodic Memory Reclamation ---
# Force GC and return memory to the OS after processing each large file
GC.gc(true)
if Sys.islinux()
ccall(:malloc_trim, Cint, (Cint,), 0)
end
current_step += 1
end
# Final memory cleanup for the batch
GC.gc(true)
if Sys.islinux()
ccall(:malloc_trim, Cint, (Cint,), 0)
end
# --- 3. Final Report ---
total_time_end = round(time() - total_time_start, digits=3)

35
build_sysimage.jl Normal file
View File

@ -0,0 +1,35 @@
# build_sysimage.jl
using PackageCompiler
println("--- Starting Custom System Image Build ---")
println("This process can take 5-10 minutes.")
# Define the image path with OS-specific extension
ext = Sys.iswindows() ? ".dll" : Sys.isapple() ? ".dylib" : ".so"
sysimage_path = "MSI_sysimage" * ext
# List dependencies to include in the sysimage for maximum stability
# We have removed graphical heavy-lifters (CairoMakie, PlotlyBase) to ensure
# the build completes successfully on systems with standard RAM.
packages_to_include = [
:Genie,
:GenieFramework,
:PlotlyBase,
:Libz,
:DataFrames,
:Serialization,
:Printf,
:JSON
]
create_sysimage(
packages_to_include,
sysimage_path = sysimage_path,
precompile_execution_file = "precompile_script.jl",
incremental = true,
filter_stdlibs = false
)
println("--- System Image Build Successful! ---")
println("To start Julia with this image, use:")
println("julia --threads auto --project=. --sysimage $(sysimage_path) start_MSI_GUI.jl")

25
julia_imzML_visual.jl
View File

@ -1131,23 +1131,24 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov
# --- Extract metadata ---
metadata = extract_metadata(local_msi_data, file_path)
# --- Save Slices ---
# --- Save Slices (Parallelized) ---
mkpath(output_dir)
for (mass_idx, mass) in enumerate(masses)
progress_message_ref = "File $(params.fileIdx)/$(params.nFiles): Saving slice for m/z=$mass"
progress_lock = ReentrantLock()
# Parallelize over the requested masses to fully utilize multi-core CPUs
Threads.@threads for mass in masses
slice = slice_dict[mass]
text_nmass = replace(string(mass), "." => "_")
bitmap_filename = params.triqE ? "TrIQ_$(text_nmass).bmp" : "MSI_$(text_nmass).bmp"
colorbar_filename = params.triqE ? "colorbar_TrIQ_$(text_nmass).png" : "colorbar_MSI_$(text_nmass).png"
local sliceQuant, bounds
if all(iszero, slice)
sliceQuant = zeros(UInt8, size(slice))
bounds = (0.0, 1.0) # Default bounds for empty slice
@warn "No intensity data for m/z = $mass in $(dataset_name)"
else
# Get both quantized data AND bounds in one call
# TrIQ and quantization are computationally intensive and now run in parallel
if params.triqE
sliceQuant, bounds = TrIQ(slice, params.colorL, params.triqP, mask_matrix=mask_matrix_for_triq)
else
@ -1156,18 +1157,24 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov
if params.medianF
sliceQuant = round.(UInt8, median_filter(sliceQuant))
# Note: bounds remain the same after median filter
end
end
# Disk I/O (Saving BMP)
save_bitmap(joinpath(output_dir, bitmap_filename), sliceQuant, ViridisPalette)
if !all(iszero, slice)
# Now pass the precomputed bounds to colorbar generation
# CairoMakie colorbar generation is now also parallelized
generate_colorbar_image(slice, params.colorL, joinpath(output_dir, colorbar_filename),
bounds; use_triq=params.triqE, triq_prob=params.triqP, mask_path=mask_path)
end
lock(progress_lock) do
progress_message_ref = "File $(params.fileIdx)/$(params.nFiles): Saved slice for m/z=$mass"
end
end
is_imzML = local_msi_data.source isa ImzMLSource
update_registry(params.registry, dataset_name, file_path, metadata, is_imzML)
return (true, "")

158
precompile_script.jl Normal file
View File

@ -0,0 +1,158 @@
# precompile_script.jl
# This script exercises the core MSI processing kernels (imzML and mzML)
# to ensure they are precompiled into the custom system image.
using MSI_src
using Mmap
using Base64
using PlotlyBase
# --- DYNAMIC WARM-UP WORKLOAD --- #
function create_warmup_datasets(dir, T_mz::Type, T_int::Type)
# 1. Create minimal imzML/ibd
imzml_path = joinpath(dir, "warmup_$(T_mz)_$(T_int).imzML")
ibd_path = joinpath(dir, "warmup_$(T_mz)_$(T_int).ibd")
n_points = 10
n_pixels = 1
mz_bytes = n_points * sizeof(T_mz)
int_bytes = n_points * sizeof(T_int)
# Write some dummy binary data
open(ibd_path, "w") do f
write(f, rand(T_mz, n_points))
write(f, rand(T_int, n_points))
end
# Minimal but VALID imzML structure that passes axes_config_img
imzml_content = """<?xml version="1.0" encoding="utf-8"?>
<mzML xmlns="http://psi.hupo.org/ms/mzML" version="1.1.0">
<cvList count="2">
<cv id="MS" fullName="Proteomics Standards Initiative Mass Spectrometry Ontology" version="3.30.0" URI="http://psidev.cvs.sourceforge.net/*checkout*/psidev/ms/mzML/ontology/psi-ms.obo"/>
<cv id="IMS" fullName="Imaging MS Ontology" version="0.9.1" URI="http://www.imzml.org/ontology/imzML1.1.0.obo"/>
</cvList>
<fileDescription>
<fileContent>
<cvParam cvRef="IMS" accession="IMS:1000031" name="processed" value=""/>
<cvParam cvRef="IMS" accession="IMS:1000080" name="universally unique identifier" value="00000000-0000-0000-0000-000000000000"/>
<cvParam cvRef="IMS" accession="IMS:1000091" name="ibd MD5 HTTP" value="00000000000000000000000000000000"/>
</fileContent>
</fileDescription>
<referenceableParamGroupList count="2">
<referenceableParamGroup id="mz_array_settings">
<cvParam cvRef="MS" accession="MS:1000514" name="m/z array" value=""/>
<cvParam cvRef="MS" accession="MS:$(T_mz == Float64 ? "1000523" : "1000521")" name="$(T_mz == Float64 ? "64-bit float" : "32-bit float")" value=""/>
<cvParam cvRef="MS" accession="MS:1000574" name="zlib compression" value=""/>
</referenceableParamGroup>
<referenceableParamGroup id="intensity_array_settings">
<cvParam cvRef="MS" accession="MS:1000515" name="intensity array" value=""/>
<cvParam cvRef="MS" accession="MS:$(T_int == Float64 ? "1000523" : "1000521")" name="$(T_int == Float64 ? "64-bit float" : "32-bit float")" value=""/>
<cvParam cvRef="MS" accession="MS:1000574" name="zlib compression" value=""/>
</referenceableParamGroup>
</referenceableParamGroupList>
<run id="run_1">
<spectrumList count="1">
<spectrum index="0" id="scan=1" defaultArrayLength="$n_points">
<cvParam cvRef="IMS" accession="IMS:1000050" name="coordinate x" value="1"/>
<cvParam cvRef="IMS" accession="IMS:1000051" name="coordinate y" value="1"/>
<binaryDataArrayList count="2">
<binaryDataArray encodedLength="0">
<referenceableParamGroupRef ref="mz_array_settings"/>
<cvParam cvRef="IMS" accession="IMS:1000102" name="external data" value=""/>
<cvParam cvRef="IMS" accession="IMS:1000103" name="external offset" value="0"/>
<cvParam cvRef="IMS" accession="IMS:1000104" name="external array length" value="$n_points"/>
<cvParam cvRef="IMS" accession="IMS:1000101" name="external encoded length" value="$mz_bytes"/>
<binary/>
</binaryDataArray>
<binaryDataArray encodedLength="0">
<referenceableParamGroupRef ref="intensity_array_settings"/>
<cvParam cvRef="IMS" accession="IMS:1000102" name="external data" value=""/>
<cvParam cvRef="IMS" accession="IMS:1000103" name="external offset" value="$mz_bytes"/>
<cvParam cvRef="IMS" accession="IMS:1000104" name="external array length" value="$n_points"/>
<cvParam cvRef="IMS" accession="IMS:1000101" name="external encoded length" value="$int_bytes"/>
<binary/>
</binaryDataArray>
</binaryDataArrayList>
</spectrum>
</spectrumList>
</run>
</mzML>
"""
write(imzml_path, imzml_content)
# 2. Create minimal mzML (Base64)
mzml_path = joinpath(dir, "warmup_$(T_mz)_$(T_int).mzML")
b64_mz = base64encode(rand(T_mz, n_points))
b64_int = base64encode(rand(T_int, n_points))
mzml_content = """<?xml version="1.0" encoding="utf-8"?>
<mzML xmlns="http://psi.hupo.org/ms/mzML" version="1.1.0">
<run id="run_1">
<spectrumList count="1">
<spectrum index="0" id="scan=1" defaultArrayLength="$n_points">
<binaryDataArrayList count="2">
<binaryDataArray encodedLength="$(length(b64_mz))">
<cvParam cvRef="MS" accession="MS:1000514" name="m/z array" value=""/>
<cvParam cvRef="MS" accession="MS:$(T_mz == Float64 ? "1000523" : "1000521")" name="" value=""/>
<binary>$b64_mz</binary>
</binaryDataArray>
<binaryDataArray encodedLength="$(length(b64_int))">
<cvParam cvRef="MS" accession="MS:1000515" name="intensity array" value=""/>
<cvParam cvRef="MS" accession="MS:$(T_int == Float64 ? "1000523" : "1000521")" name="" value=""/>
<binary>$b64_int</binary>
</binaryDataArray>
</binaryDataArrayList>
</spectrum>
</spectrumList>
</run>
<indexList count="1"><index name="spectrum"><offset idRef="scan=1">PLACEHOLDER</offset></index></indexList>
<indexListOffset>ID_OFFSET</indexListOffset>
</mzML>
"""
spec_start = findfirst("<spectrum", mzml_content).start - 1
index_start = findfirst("<indexList", mzml_content).start - 1
mzml_content = replace(mzml_content, "PLACEHOLDER" => string(spec_start))
mzml_content = replace(mzml_content, "ID_OFFSET" => string(index_start))
write(mzml_path, mzml_content)
return imzml_path, mzml_path
end
println("Starting robust precompilation warmup (imzML + mzML)...")
try
mktempdir() do tmp_dir
for (T_mz, T_int) in [(Float32, Float32), (Float64, Float32)]
println("Exercising pathways for $T_mz and $T_int...")
imzml_path, mzml_path = create_warmup_datasets(tmp_dir, T_mz, T_int)
try
# Exercise imzML pathway
imzml_data = MSI_src.load_imzml_lazy(imzml_path; use_mmap=true)
MSI_src.get_multiple_mz_slices(imzml_data, [10.0, 20.0], 0.1)
# Exercise mzML pathway
mzml_data = MSI_src.load_mzml_lazy(mzml_path)
MSI_src.GetSpectrum(mzml_data, 1)
catch e
@warn "Warmup failed for $T_mz/$T_int: $e"
end
end
# --- COMMON KERNELS ---
println("Precompiling image processing kernels...")
dummy_slice = rand(Float32, 4, 4)
MSI_src.TrIQ(dummy_slice, 256, 1.0)
MSI_src.save_bitmap(joinpath(tmp_dir, "d.bmp"), zeros(UInt8, 4, 4), MSI_src.ViridisPalette)
# Plotly Warmup
p = PlotlyBase.Plot(PlotlyBase.scatter(x=1:2, y=[1,2]))
end
catch e
@warn "Global Warmup failed: $e"
catch
# Silent fail for interrupt
end
println("Precompilation workload completed.")