From 70fff16170ca5784651f408bc9e311477300fca4 Mon Sep 17 00:00:00 2001 From: Pixelguy14 Date: Tue, 7 Apr 2026 14:06:02 -0600 Subject: [PATCH] more secure locks and a script to create a system image of juliaMSI for faster boot times, still experimental but everything else remains pristine --- .gitignore | 4 +- Project.toml | 5 +- README.md | 34 ++++++++- app.jl | 14 ++++ build_sysimage.jl | 35 ++++++++++ julia_imzML_visual.jl | 25 ++++--- precompile_script.jl | 158 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 262 insertions(+), 13 deletions(-) create mode 100644 build_sysimage.jl create mode 100644 precompile_script.jl diff --git a/.gitignore b/.gitignore index 0bec6dd..b219dd4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ log/* R_original_scripts/ test/results/ test/binarization_results/ -Manifest.toml \ No newline at end of file +Manifest.toml +MSI_sysimage.dll +MSI_sysimage.so \ No newline at end of file diff --git a/Project.toml b/Project.toml index 72450c3..48a8ef4 100644 --- a/Project.toml +++ b/Project.toml @@ -35,6 +35,7 @@ Loess = "4345ca2d-374a-55d4-8d30-97f9976e7612" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4" NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85" +PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a" PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" @@ -52,9 +53,9 @@ Accessors = "0.1" Base64 = "1.11" CSV = "0.10" CairoMakie = "0.13" +CodecBase = "0.3" ColorSchemes = "3.30" Colors = "0.12" -CodecBase = "0.3" ContinuousWavelets = "1.1" DataFrames = "1.7" Dates = "1.11" @@ -63,7 +64,6 @@ GLMakie = "0.11" Genie = "5.31" GenieFramework = "2.8" HistogramThresholding = "0.3" -Interpolations = "0.15" ImageBinarization = "0.3" ImageComponentAnalysis = "0.2" ImageContrastAdjustment = "0.3" @@ -72,6 +72,7 @@ ImageFiltering = "0.7" ImageMorphology = "0.4" ImageSegmentation = "1.9" Images = "0.26" +Interpolations = "0.15" JLD2 = "0.6" JSON = "0.21" Libz = "1.0" diff --git a/README.md b/README.md index f63ed57..e83f4b9 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ https://codeberg.org/LabABI/JuliaMSI Example of a correct route:
~/Downloads/JuliaMSI-main/juliamsi 2. Without entering the Julia environment, launch the project in your terminal with the following command (which works for all operating systems): - ``` + ```bash julia --threads auto --project=. start_MSI_GUI.jl ``` 3. After the script has finished loading, you can open a [page](http://127.0.0.1:1481/) in your browser with the web app running. @@ -34,6 +34,38 @@ Minimum system requirements: 4 core processor, 8 GB RAM
JuliaMSI is a Graphical User Interface for a library of MSI tools in Julia: https://github.com/CINVESTAV-LABI/julia_mzML_imzML +## Build the System Image (One-time) +Alternatively, you can generate the .so file by running the next build script in your directory: + +```bash +julia --project=. build_sysimage.jl +``` +This may take 5–10 minutes as it merges all dependencies and JuliaMSI functions into a single binary. The resulting .so/.dll file will be around **300MB–600MB** because it contains the pre-compiled machine code for your entire environment. +A sysimage built on Linux (.so) will not work on Windows. +You must run the build_sysimage.jl script once on each target operating system. + +Once MSI_sysimage.so is created in your directory, adapt your command like this: +```bash +julia --project=. -e 'using Pkg; Pkg.precompile()' +# This will find MSI_sysimage.so, .dll, or .dylib automatically: +julia --threads auto --project=. --sysimage MSI_sysimage* start_MSI_GUI.jl +``` +The command above loads a pre-compiled version of your environment (note, you have to use either the sysimage command or the normal start command), speeding up the booting delay. Your GUI or scripts using JuliaMSI should start almost instantly and process files significantly faster than before. +Always run the build script **in the same project root** where you intend to run the app. + +In scripts like test/run_tests.jl, you no longer need to manually include("../src/MSI_src.jl"). You can simply treat it like a globally installed package to load the precompiled binary: + +```julia +using MSI_src +# ... rest of your script +``` +### Run with the --sysimage flag +Launch your tests using the same flag to bypass all compilation overhead: + +```bash +julia --threads auto --project=. --sysimage MSI_sysimage.so test/run_tests.jl +``` + ## License JuliaMSI is published under the terms of the MIT License. diff --git a/app.jl b/app.jl index 11058a5..f4bdec4 100644 --- a/app.jl +++ b/app.jl @@ -2422,8 +2422,22 @@ end else push!(newly_created_folders, replace(basename(file_path), r"\.imzML$"i => "")) end + + # --- Periodic Memory Reclamation --- + # Force GC and return memory to the OS after processing each large file + GC.gc(true) + if Sys.islinux() + ccall(:malloc_trim, Cint, (Cint,), 0) + end + current_step += 1 end + + # Final memory cleanup for the batch + GC.gc(true) + if Sys.islinux() + ccall(:malloc_trim, Cint, (Cint,), 0) + end # --- 3. Final Report --- total_time_end = round(time() - total_time_start, digits=3) diff --git a/build_sysimage.jl b/build_sysimage.jl new file mode 100644 index 0000000..f1c8e43 --- /dev/null +++ b/build_sysimage.jl @@ -0,0 +1,35 @@ +# build_sysimage.jl +using PackageCompiler + +println("--- Starting Custom System Image Build ---") +println("This process can take 5-10 minutes.") + +# Define the image path with OS-specific extension +ext = Sys.iswindows() ? ".dll" : Sys.isapple() ? ".dylib" : ".so" +sysimage_path = "MSI_sysimage" * ext + +# List dependencies to include in the sysimage for maximum stability +# We have removed graphical heavy-lifters (CairoMakie, PlotlyBase) to ensure +# the build completes successfully on systems with standard RAM. +packages_to_include = [ + :Genie, + :GenieFramework, + :PlotlyBase, + :Libz, + :DataFrames, + :Serialization, + :Printf, + :JSON +] + +create_sysimage( + packages_to_include, + sysimage_path = sysimage_path, + precompile_execution_file = "precompile_script.jl", + incremental = true, + filter_stdlibs = false +) + +println("--- System Image Build Successful! ---") +println("To start Julia with this image, use:") +println("julia --threads auto --project=. --sysimage $(sysimage_path) start_MSI_GUI.jl") diff --git a/julia_imzML_visual.jl b/julia_imzML_visual.jl index 33acb19..95ae054 100644 --- a/julia_imzML_visual.jl +++ b/julia_imzML_visual.jl @@ -1131,23 +1131,24 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov # --- Extract metadata --- metadata = extract_metadata(local_msi_data, file_path) - # --- Save Slices --- + # --- Save Slices (Parallelized) --- mkpath(output_dir) - - for (mass_idx, mass) in enumerate(masses) - progress_message_ref = "File $(params.fileIdx)/$(params.nFiles): Saving slice for m/z=$mass" - + + progress_lock = ReentrantLock() + + # Parallelize over the requested masses to fully utilize multi-core CPUs + Threads.@threads for mass in masses slice = slice_dict[mass] text_nmass = replace(string(mass), "." => "_") bitmap_filename = params.triqE ? "TrIQ_$(text_nmass).bmp" : "MSI_$(text_nmass).bmp" colorbar_filename = params.triqE ? "colorbar_TrIQ_$(text_nmass).png" : "colorbar_MSI_$(text_nmass).png" + local sliceQuant, bounds if all(iszero, slice) sliceQuant = zeros(UInt8, size(slice)) bounds = (0.0, 1.0) # Default bounds for empty slice - @warn "No intensity data for m/z = $mass in $(dataset_name)" else - # Get both quantized data AND bounds in one call + # TrIQ and quantization are computationally intensive and now run in parallel if params.triqE sliceQuant, bounds = TrIQ(slice, params.colorL, params.triqP, mask_matrix=mask_matrix_for_triq) else @@ -1156,18 +1157,24 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov if params.medianF sliceQuant = round.(UInt8, median_filter(sliceQuant)) - # Note: bounds remain the same after median filter end end + # Disk I/O (Saving BMP) save_bitmap(joinpath(output_dir, bitmap_filename), sliceQuant, ViridisPalette) + if !all(iszero, slice) - # Now pass the precomputed bounds to colorbar generation + # CairoMakie colorbar generation is now also parallelized generate_colorbar_image(slice, params.colorL, joinpath(output_dir, colorbar_filename), bounds; use_triq=params.triqE, triq_prob=params.triqP, mask_path=mask_path) end + + lock(progress_lock) do + progress_message_ref = "File $(params.fileIdx)/$(params.nFiles): Saved slice for m/z=$mass" + end end + is_imzML = local_msi_data.source isa ImzMLSource update_registry(params.registry, dataset_name, file_path, metadata, is_imzML) return (true, "") diff --git a/precompile_script.jl b/precompile_script.jl new file mode 100644 index 0000000..050df9a --- /dev/null +++ b/precompile_script.jl @@ -0,0 +1,158 @@ +# precompile_script.jl +# This script exercises the core MSI processing kernels (imzML and mzML) +# to ensure they are precompiled into the custom system image. + +using MSI_src +using Mmap +using Base64 +using PlotlyBase + +# --- DYNAMIC WARM-UP WORKLOAD --- # + +function create_warmup_datasets(dir, T_mz::Type, T_int::Type) + # 1. Create minimal imzML/ibd + imzml_path = joinpath(dir, "warmup_$(T_mz)_$(T_int).imzML") + ibd_path = joinpath(dir, "warmup_$(T_mz)_$(T_int).ibd") + + n_points = 10 + n_pixels = 1 + mz_bytes = n_points * sizeof(T_mz) + int_bytes = n_points * sizeof(T_int) + + # Write some dummy binary data + open(ibd_path, "w") do f + write(f, rand(T_mz, n_points)) + write(f, rand(T_int, n_points)) + end + + # Minimal but VALID imzML structure that passes axes_config_img + imzml_content = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + write(imzml_path, imzml_content) + + # 2. Create minimal mzML (Base64) + mzml_path = joinpath(dir, "warmup_$(T_mz)_$(T_int).mzML") + b64_mz = base64encode(rand(T_mz, n_points)) + b64_int = base64encode(rand(T_int, n_points)) + + mzml_content = """ + + + + + + + + + $b64_mz + + + + + $b64_int + + + + + + PLACEHOLDER + ID_OFFSET + +""" + spec_start = findfirst(" string(spec_start)) + mzml_content = replace(mzml_content, "ID_OFFSET" => string(index_start)) + + write(mzml_path, mzml_content) + + return imzml_path, mzml_path +end + +println("Starting robust precompilation warmup (imzML + mzML)...") + +try + mktempdir() do tmp_dir + for (T_mz, T_int) in [(Float32, Float32), (Float64, Float32)] + println("Exercising pathways for $T_mz and $T_int...") + imzml_path, mzml_path = create_warmup_datasets(tmp_dir, T_mz, T_int) + + try + # Exercise imzML pathway + imzml_data = MSI_src.load_imzml_lazy(imzml_path; use_mmap=true) + MSI_src.get_multiple_mz_slices(imzml_data, [10.0, 20.0], 0.1) + + # Exercise mzML pathway + mzml_data = MSI_src.load_mzml_lazy(mzml_path) + MSI_src.GetSpectrum(mzml_data, 1) + catch e + @warn "Warmup failed for $T_mz/$T_int: $e" + end + end + + # --- COMMON KERNELS --- + println("Precompiling image processing kernels...") + dummy_slice = rand(Float32, 4, 4) + MSI_src.TrIQ(dummy_slice, 256, 1.0) + MSI_src.save_bitmap(joinpath(tmp_dir, "d.bmp"), zeros(UInt8, 4, 4), MSI_src.ViridisPalette) + + # Plotly Warmup + p = PlotlyBase.Plot(PlotlyBase.scatter(x=1:2, y=[1,2])) + end +catch e + @warn "Global Warmup failed: $e" +catch + # Silent fail for interrupt +end +println("Precompilation workload completed.")