Merge pull request 'dev/Memory_pooling_adjustments' (#1) from dev/Memory_pooling_adjustments into main

Reviewed-on: #1

.github/workflows/ci.yml

@@ -0,0 +1,32 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1.10'
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macOS-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/cache@v2
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1

.gitignore

@@ -7,3 +7,6 @@ log/*
 R_original_scripts/
 test/results/
 test/binarization_results/
+Manifest.toml
+MSI_sysimage.dll
+MSI_sysimage.so

Dockerfile.headless

@@ -0,0 +1,31 @@
+# Dockerfile.headless
+# This Dockerfile creates a minimal, high-speed container solely intended
+# for running JuliaMSI scripts via the pre-compiled headless sysimage.
+# It explicitly excludes the Genie UI and graphical routing to optimize size and start speed.
+
+FROM julia:1.11
+
+# System dependencies for core MSI math libraries 
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Only copy essential library files (no UI or app.jl required)
+COPY Project.toml Manifest.toml ./
+COPY src/ ./src/
+COPY build_sysimage.jl precompile_script.jl ./
+COPY test/ ./test/
+
+# Instantiate the environment
+RUN julia --project=. -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+
+# Build the headless sysimage
+# This command strips out Genie and plot libraries, creating a pure data-engine .so
+RUN julia --project=. build_sysimage.jl --headless
+
+# The resulting image is heavily optimized for zero-JIT execution.
+# Default command simply enters a fast REPL.
+CMD ["julia", "--project=.", "--threads", "auto", "-J", "sys_msi_headless.so"]

JuliaMSI_MainGuide.md

@@ -0,0 +1,217 @@
+# JuliaMSI Platform User Guide
+## Core Workflow and Interactive Analysis
+
+---
+
+### Table of Contents
+1. [Introduction](#introduction)
+2. [Loading a Dataset](#loading-a-dataset)
+3. [Converting mzML to imzML](#converting-mzml-to-imzml)
+4. [Batch Processing](#batch-processing)
+5. [Spectral Visualization](#spectral-visualization)
+6. [Imaging Slice Generation](#imaging-slice-generation)
+7. [Topography and Surface Plots](#topography-and-surface-plots)
+8. [Small Features and Tools](#small-features-and-tools)
+    * [Metadata Viewing](#metadata-viewing)
+    * [Optical Image Overlay](#optical-image-overlay)
+    * [Comparison of Datasets](#comparison-of-datasets)
+    * [Registry and Safe Processing](#registry-and-safe-processing)
+9. [Mask Generation and Usage (mask.jl)](#mask-generation-and-usage-maskjl)
+10. [Preprocessing](#preprocessing)
+11. [Practical Tips for Analysts](#practical-tips-for-analysts)
+    * [Julia Version Compatibility](#julia-version-compatibility)
+    * [First-Time Startup](#first-time-startup)
+    * [Accessing the Web Interface](#accessing-the-web-interface)
+    * [Memory Management](#memory-management)
+    * [Registry and File Locations](#registry-and-file-locations)
+    * [Mask Handling](#mask-handling)
+12. [Troubleshooting Common Issues](#troubleshooting-common-issues)
+13. [Citation and Contact](#citation-and-contact)
+
+---
+
+## Introduction
+This document serves as an entry point for analysts using the **JuliaMSI** platform. It focuses on the user-facing modules: `app.jl` (the main application entry), `julia_imzml_visual.jl` (visualization and heavy procedures), and `mask.jl` (mask generation and filtering). The goal is to explain why each step requires specific inputs and what those inputs mean scientifically.
+
+**JuliaMSI** is a user-friendly Graphical User Interface (GUI) developed in Julia for MSI data analyses. Our framework allows for rapid **imzML** MSI data analysis and **mzML** spectra plot loading, significantly improving analytical workflows. High-resolution instruments produce massive datasets, requiring efficient memory management and parallel processing—reasons why JuliaMSI is written in Julia, which offers high performance.
+
+### Core Objectives:
+*   **Simplification**: Reducing command-line dependency through intuitive visualizations and easy parameter input.
+*   **Accessibility**: Breaking the proprietary software barrier across fields like plant sciences, biomedical research, and environmental studies.
+*   **Interactive Exploration**: Using the `Plotly.jl` package for dynamic visualizations. Users can explore multiple interpretations of images, select specific coordinates for spectra analysis, or pick an *m/z* value directly from the displayed spectrum.
+
+### Key Capabilities:
+*   **Smart data handling**: Processes large files in manageable sections without overwhelming memory.
+*   **Batch processing**: Analyzes multiple samples simultaneously for high-throughput studies.
+*   **Interactive exploration**: Web interface enables real-time data visualization and analysis.
+
+---
+
+## Loading a Dataset
+**Action**: Click the **Search** button (`@onbutton btnSearch`)
+
+Load an **imzML** or **mzML** file into the workspace, extract metadata, and prepare the dataset for visualization and processing.
+
+The platform checks whether a pre-processed version of the same file exists in the registry (**fast load**). If not, it performs a **full load** using `OpenMSIData`. The GUI handles multiple file uploads for MSI slice generation; however, for spectra and preprocessing, it will only use the last selected file.
+
+---
+
+## Converting mzML to imzML
+**Action**: Click **Convert/Process** (`@onbutton convert_process`)
+
+This tool converts an **mzML** file (from a non-imaging mass spectrometry run) into an **imzML** file suitable for imaging analysis. This is necessary when spectra were acquired in a raster pattern but saved as a continuous **mzML** file without spatial coordinates.
+
+### Inputs Required:
+*   **mzML file**: The raw mass spectrometry data file.
+*   **Sync file**: A text file that maps each spectrum in the **mzML** to a specific pixel coordinate (*x, y*). The sync file typically contains one line per spectrum with the format `x y` or similar.
+
+> **Why these inputs?**: **mzML** files lack spatial metadata; they contain a list of spectra in acquisition order. Without coordinate information, the data cannot be reconstructed as an image. **Sync files** provide the missing spatial mapping, often generated by instrument software from a motor stage log. The conversion process embeds these coordinates into the **imzML** structure.
+
+---
+
+## Batch Processing
+**Action**: Click **Main Process** (`@onbutton mainProcess`)
+
+This is the core image generation routine. It takes a list of comma-separated *m/z* values and produces ion images (and optionally **TrIQ Quantized** images) for each file loaded.
+
+### Required Inputs:
+*   **m/z values**: A comma-separated list of *m/z* values to visualize spatial distribution. These are the core of MSI: each image maps the abundance of a particular ion across the sample. Without a target, there is no image.
+*   **Tolerance**: The mass window around each target ion. Necessary because no mass spectrometer measures masses with infinite precision. The signal from a given ion is spread over a small *m/z* range; integrating over that range (by summing intensities of all data points within ±tolerance) captures the true signal.
+*   **Color level**: Number of discrete color levels in the output BMP image. A low number (e.g., 16) gives a coarse view; a high number (256) shows subtle intensity variations but can be noisy.
+*   **TrIQ**: A toggle to enable **Threshold Intensity Quantization**. This produces images with enhanced contrast and reduced influence of extreme outliers, making spatial patterns more visible. It is a contrast normalization and quantization technique.
+*   **TrIQ Threshold**: Probability threshold for outlier removal in TrIQ.
+*   **Median filter**: Replaces each pixel’s intensity with the median of its neighbors, effectively removing isolated "hot pixels" while preserving sharp boundaries.
+*   **Mask**: Toggle to focus on biologically relevant regions (e.g., tissue vs. background), avoiding wasted computational resources on irrelevant pixels.
+
+> **Note**: The mask path is determined inside the JSON file, which can be manually altered, or using the specific section in `mask.jl`.
+
+---
+
+## Imaging Slice Generation (Part of Main Process)
+In MSI, each pixel contains a full mass spectrum. To create an ion image (or "slice"), we must decide which data points in that spectrum belong to the ion of interest.
+
+*   **Tolerance**: Defines the *m/z* window. Too narrow may miss part of the peak; too wide includes noise or interfering peaks. Optimal tolerance depends on instrument resolving power and calibration stability.
+*   **Colorbar Generation**: Each image is accompanied by a colorbar PNG showing the mapping between intensity and colors. The bounds are determined during quantization (TrIQ or min-max scaling) and passed to `generate_colorbar_image`. This ensures consistent color interpretation with labels indicating the intensity range and scaling factors (e.g., "Intensity ×10³").
+
+---
+
+## Spectral Visualization
+**Functions**: `createMeanPlot`, `createSumPlot`, `createXYPlot`, `createNSpectrumPlot`
+
+Displays spectra from the loaded dataset, optionally filtered by a mask. This generates a Plotly layout plot for quality control, identification, and *m/z* selection.
+
+### Inputs & Features:
+*   **Dataset**: The last **imzML** / **mzML** file loaded in the GUI.
+*   **Mask**: Toggle for mask filtering. The spectrum is computed only from pixels inside the mask.
+*   **User click**: If a user clicks on an already generated **slice** (2D image), two vertical lines appear and values are filled (one for *x*-axis, one for *y*-axis) to generate spectrum plot indicating the position of the click in the image. This allows the creation of a spectrum plot correlating to specific spatial selection.
+*   **NSpectrumPlot**: Allows the user to select specific spectra based on the **ID** (the index of the spectra in the dataset).
+
+### Spectral Types:
+*   **Mean spectrum**: The average of all pixel spectra. Gives an overview of ions present in the entire sample and their relative abundances. Used to identify peaks for further imaging.
+*   **Sum spectrum**: Total ion current summed across all pixels. Emphasizes ions that are abundant across many pixels.
+*   **Per-pixel spectra (XY and N)**: Reveal the distribution of ions in a specific location, useful for comparing tissue regions (e.g., tumor vs. normal).
+
+> **Why these inputs?**: Spectra are the raw data of MSI. Visualizing them is the first step: checking for unexpected peaks, assessing signal-to-noise ratio, and verifying mass calibration. Applying a mask (e.g., only tumor region) allows researchers to extract region-specific molecular signatures.
+
+---
+
+## Topography and Surface Plots
+**Functions**: `image3dPlot`, `imageCPlot`, `triq3dPlot`, `triqCPlot`
+
+Convert a 2D ion image into a 3D surface plot where the height (*z*-axis) represents ion intensity.
+*   **Inputs**: Current ion image and optional Mask.
+*   **Why these inputs?**: 3D surfaces and 2D Contour plots make intensity variations more tangible, especially for features with large dynamic range. They are useful for understanding the topography of ion distributions (e.g., a lipid enriched in a specific tissue layer). The 3D plot is interactive, allowing for rotation and zoom.
+
+---
+
+## Small Features and Tools
+
+### Metadata Viewing (`showMetadataBtn`)
+Display a summary of the dataset’s metadata: instrument settings, acquisition parameters, image dimensions, number of pixels, *m/z* range, etc.
+
+### Optical Image Overlay (`btnOptical`)
+Overlay a histological or photographic image onto the MSI image to correlate molecular distributions with tissue anatomy.
+*   **Inputs**: Current ion image, Transparency slider (`imgTrans`), and Optical image file (PNG, BMP, JPG, JPEG).
+*   **Output**: The main image tab updates to show the MSI image (or TrIQ image) blended with the optical background.
+
+### Comparison of Datasets
+Functionality to compare two datasets (e.g., control vs. treated). This involves side-by-side visualization or ratio image generation to highlight differences in molecular abundance. It allows visualization of slices with or without TrIQ filtering.
+
+### Registry and Safe Processing
+Batch processing is orchestrated by `process_file_safely`. The registry (`registry.json`) stores critical info: source path, image dimensions, summary statistics, and mask path. This file can be edited manually following its generated structure.
+
+---
+
+## Mask Generation and Usage (`mask.jl`)
+Create binary masks that define **Regions of Interest (ROI)** in the sample. Masks filter spectra, preprocessing, and images to include only pixels inside the ROI.
+
+### User-Defined Inputs:
+*   **Drawing tools**: Draw freehand shapes on the ion image or optical overlay.
+*   **Otsu Scale**: Automatically generates a mask based on the Otsu thresholding algorithm with the slider percentage.
+*   **Noise Size**: Removes noise from the mask based on the slider percentage.
+*   **Hole Size**: Removes holes in the mask based on the slider percentage.
+*   **Smoothing**: Smooths the mask based on the slider percentage.
+*   **Thresholding**: Set intensity thresholds to automatically segment regions.
+*   **Import**: Load a pre-existing mask image (e.g., from a segmentation tool).
+
+### Outputs:
+*   A binary mask matrix (same dimensions as the MSI image) saved in the public/css/masks folder as a `.png` file with the name of the dataset.
+*   The mask is registered in the dataset’s metadata entry for automatic application in subsequent analyses.
+
+> **Why these inputs?**: Often only a portion of the imaged area contains tissue; the rest could be background. Masking removes background pixels, improving statistical power and avoiding artifacts. Manual delineation based on optical images is standard practice in histology-directed MSI. It can also work for a selection of an area of interest (e.g. only the tumor region) in samples with multiple areas of interest.
+
+---
+
+## Preprocessing
+To understand the deep structure of the preprocessing pipeline, please refer to the dedicated guide: **[JuliaMSI Preprocessing Guide](JuliaMSI_Preprocessing.md)**
+
+This document describes the rationale, methods (TIC/PQN, SNIP, Wavelets), and implementation choices in detail.
+
+---
+
+## Practical Tips for Analysts
+
+### Julia Version Compatibility
+*   The platform is tested and compatible with **Julia 1.11**.
+*   **Recommendation**: Use Julia 1.11 to ensure all dependencies work correctly. Use **Juliaup** to manage multiple versions easily. Downgrading your system Julia is unnecessary; you can keep multiple versions and launch the platform with the specific executable path.
+
+### First-Time Startup
+*   The script will detect a missing `Manifest.toml` and automatically instantiate packages. This may take a few minutes. Subsequent startups will be much faster.
+*   **Caution**: Do not close the terminal/console while the server is running; it hosts the web interface.
+
+### Accessing the Web Interface
+*   The platform opens your browser to `http://127.0.0.1:1481`.
+*   If port 1481 is in use, change this segment in the startup script: `up(host="127.0.0.1", port=1481)`.
+
+### Memory Management
+*   MSI datasets can be large (gigabytes). JuliaMSI includes automatic **garbage collection** and, on Linux, calls `malloc_trim` to return freed memory to the OS.
+*   **OOM Troubleshooting**: Process fewer *m/z* values at once, use a mask to exclude background, or increase system swap space.
+
+### Registry and File Locations
+*   `registry.json` stores paths to your datasets. If you move original **imzML** files, update the paths in the JSON or delete the registry file to let the platform rebuild it.
+*   Processed images and colorbars are saved in the `public/` folder, organized by dataset name.
+
+### Mask Handling
+*   Ensure manually created masks have the **exact same dimensions** (width × height) as the MSI image.
+*   Masks are automatically applied during batch processing if the "Mask" toggle is enabled. If you move the mask file, you must update its path in the registry.
+
+---
+
+## Troubleshooting Common Issues
+
+*   **"Error loading active file"**: Check that the file is a valid **imzML** or **mzML** and that the corresponding `.ibd` file is in the same directory.
+*   **Images appear all black**: This can happen if the tolerance is too narrow (no signal captured) or if the mask is incorrectly applied. Verify *m/z* and tolerance.
+*   **Slow performance**: Reduce the number of *m/z* values or disable the median filter. Ensure you have enough RAM.
+*   **Browser interface not responding**: Refresh the page. Check the console for errors; you may need to restart the Julia process.
+
+---
+
+## Citation and Contact
+Any revision to the code, bug, or error should be reported to: **julian.sierrag@icloud.com**
+
+*Note: This guide is based on collective knowledge of these algorithms; please perform your own research and cite properly. To cite the JuliaMSI platform, please use:*
+
+> José Julián Sierra-Álvarez, Martín Orlando Camargo-Escalante, Carlos Daniel Sierra-Álvarez, Carmelo Hernández-Caricio, Juan Francisco Moreno-Luna, Isabel Buendía-Corona, Robert Winkler,
+> **JuliaMSI: A high-performance graphical platform for mass spectrometry imaging data analysis**,
+> *Analytica Chimica Acta*, Volume 1377, 2025, 344613, ISSN 0003-2670,
+> [https://doi.org/10.1016/j.aca.2025.344613](https://doi.org/10.1016/j.aca.2025.344613)

JuliaMSI_Preprocessing.md

@@ -0,0 +1,354 @@
+# A Comprehensive Guide to Mass Spectrometry Imaging Preprocessing
+## Rationale, Methods, and Implementation
+
+---
+
+### Table of Contents
+1. [Introduction](#introduction)
+2. [Profile vs. Centroid Mode Data](#profile-vs-centroid-mode-data)
+3. [General Principles of MSI Preprocessing](#general-principles-of-msi-preprocessing)
+4. [Step-by-Step Breakdown of Preprocessing Steps](#step-by-step-breakdown-of-preprocessing-steps)
+    * [Validation and Quality Control](#validation-and-quality-control)
+    * [Intensity Transformation (Stabilization)](#intensity-transformation-stabilization)
+    * [Smoothing](#smoothing)
+    * [Baseline Correction](#baseline-correction)
+    * [Normalization](#normalization)
+    * [Peak Detection](#peak-detection)
+    * [Peak Selection (Filtering)](#peak-selection-filtering)
+    * [Calibration](#calibration)
+    * [Peak Alignment](#peak-alignment)
+    * [Peak Binning and Feature Matrix Generation](#peak-binning-and-feature-matrix-generation)
+5. [Example Walkthrough: From Raw Spectra to Feature Matrix](#example-walkthrough-from-raw-spectra-to-feature-matrix)
+    * [Phase 1 – Instrument & Data Characteristics](#phase-1--instrument--data-characteristics)
+    * [Phase 2 – Noise & Signal Quality](#phase-2--noise--signal-quality)
+    * [Phase 3 – Mass Accuracy Analysis](#phase-3--mass-accuracy-analysis)
+    * [Phase 4 – Spatial Region Analysis](#phase-4--spatial-region-analysis)
+    * [Phase 5 – Peak Characteristics](#phase-5--peak-characteristics)
+    * [Phase 6 – Preprocessing Recommendations](#phase-6--preprocessing-recommendations)
+6. [Extra Tips for the Analyst](#extra-tips-for-the-analyst)
+    * [Isotopic and Charge State Matters](#isotopic-and-charge-state-matters)
+    * [Always Visualize the Average Spectrum First](#always-visualize-the-average-spectrum-first)
+    * [Preprocessing Is Not a One-Shot Process](#preprocessing-is-not-a-one-shot-process)
+
+---
+
+## Introduction
+Mass Spectrometry Imaging (MSI) generates thousands of mass spectra, each acquired from a distinct pixel location. In the literature, before any interpretation can be drawn from these data, a series of preprocessing steps must be applied. These steps correct for instrumental artifacts, reduce noise, and align data across pixels to produce a clean, comparable feature matrix suitable for statistical analysis and image reconstruction.
+
+Inside the **JuliaMSI** code (`Preprocessing.jl` and `PreprocessingPipeline.jl`), we have a preprocessing workflow inspired by established R packages like **MALDIquant** and **Cardinal**.
+
+This document explains the rationale behind each step, the required inputs, the scientific meaning of parameters, and the context behind key algorithms such as **TIC normalization**, **PQN**, **wavelet-based peak detection**, and **LOWESS alignment**.
+
+---
+
+## Profile vs. Centroid Mode Data
+
+Mass spectrometry data can be acquired in two primary modes: **profile** and **centroid**. Understanding the difference is crucial because preprocessing steps must be adapted accordingly.
+
+*   **Profile mode**:
+    The instrument records intensity at every digitized point along the *m/z* axis, producing a continuous waveform. Peaks appear as Gaussian-like shapes. This mode retains full peak shape information, which is valuable for accurate mass measurement, resolution assessment, and advanced peak detection. However, file sizes are large and noise is present between peaks.
+    
+*   **Centroid mode**:
+    The instrument performs real-time peak detection and reduces each peak to a single centroid (*m/z*, intensity) pair, discarding the underlying profile. This dramatically reduces file size but loses peak shape details. Centroid data are essentially a list of detected peaks with no information about peak width or baseline.
+
+In **profile mode**, algorithms must operate on the full *m/z* and intensity vectors, performing tasks like smoothing, baseline correction, and peak picking on continuous data. In **centroid mode**, these steps are often unnecessary or simplified: smoothing and baseline correction are not applicable because the data are already peak lists; peak picking is replaced by filtering based on intensity thresholds. 
+
+> [!IMPORTANT]
+> The JuliaMSI code internally distinguishes this difference in some steps by acquiring the mode from the metadata on the files, but it's up to the analyst to decide skipping smoothing/baseline steps for centroid data, for example.
+
+---
+
+## General Principles of MSI Preprocessing
+
+All preprocessing steps share a common goal: **to end up with an aligned feature matrix where rows correspond to pixels (spectra) and columns correspond to *m/z* bins (features).**
+
+The steps are applied sequentially, often in a specific order to avoid compounding errors. For example, **baseline correction should precede normalization** because baseline offsets can distort total ion current calculations. Similarly, **peak alignment should follow peak detection** because alignment requires a set of common peaks across spectra.
+
+---
+
+## Step-by-Step Breakdown of Preprocessing Steps
+
+### Validation and Quality Control
+*(This step doesn't require analyst input)*
+*   **Functions in code**: `validate_spectrum`, `qc_is_empty`, `qc_is_regular`
+*   **Purpose**: Ensure that each input spectrum is well-formed and contains usable data before any processing.
+*   **Inputs required**:
+    *   `mz::AbstractVector{<:Real}`: The *m/z* values.
+    *   `intensity::AbstractVector{<:Real}`: The corresponding intensities.
+
+> **Why these inputs?**
+> Every subsequent algorithm depends on the integrity of these vectors. Checking for non-emptiness, equal length, finite values, non-negativity, and monotonic *m/z* order prevents cryptic errors later.
+
+---
+
+### Intensity Transformation (Stabilization)
+*   **Function**: `transform_intensity_core`
+*   **Purpose**: Apply a variance-stabilizing transformation to make the data more homoscedastic (constant variance across intensity range).
+*   **Inputs required**:
+    *   `intensity::AbstractVector{<:Real}`
+    *   `method::Symbol` (e.g., `:sqrt`, `:log1p`)
+
+> **Why these inputs?**
+> The transformation operates on the intensity vector only; *m/z* values remain unchanged. The choice of method depends on the data characteristics (e.g., presence of zeros).
+>
+> Stabilization uses functions like **square root**, **log**, **log2**, **log10** and **log1p** helps the analyst stabilize variance for count data, or handle quadratical variance. The analyst must know which one of these methods to use according to the data they are visualizing (e.g., by the mean spectrum plot, or a plot from a specific coordinate).
+>
+> In MSI in general, **square root stabilization** is common because ion counting statistics approximate Poisson.
+>
+> **Example**: A spectrum with intense peaks and low baseline noise: after square root, the dynamic range is compressed, making low-intensity features more comparable to high-intensity ones.
+
+---
+
+### Smoothing
+*   **Function**: `smooth_spectrum_core`, `moving_average_smooth`
+*   **Purpose**: Reduce high-frequency noise while preserving peak shapes.
+*   **Inputs required**:
+    *   `y::AbstractVector{<:Real}`: Intensity vector.
+    *   `method::Symbol` (`:savitzky_golay`, `:moving_average`)
+    *   `window::Int`: Size of the smoothing window.
+    *   `order::Int`: Polynomial order for Savitzky–Golay.
+
+> **Why these inputs?**
+> Smoothing operates on the intensity vector only. The window size must be an odd integer for symmetry; the polynomial order must be less than the window. These constraints ensure the filter behaves properly (e.g., Savitzky–Golay requires the window bigger than the order parameter).
+>
+> **Savitzky–Golay** fits a low-degree polynomial to the local window, preserving moments of the peak (height, width) better than a simple moving average, which can distort peak shapes.
+>
+> **Example**: A noisy peak with 5% random noise: after Savitzky–Golay, the peak becomes smoother while its centroid and height remain accurate.
+
+---
+
+### Baseline Correction
+*   **Function**: `apply_baseline_correction_core`, `_snip_baseline_impl`, `convex_hull_baseline`, `median_baseline`
+*   **Purpose**: Estimate and subtract the background signal (baseline) from each spectrum.
+*   **Inputs required**:
+    *   `y::AbstractVector{<:Real}`: Intensity vector.
+    *   `method::Symbol` (`:snip`, `:convex_hull`, `:median`)
+    *   `iterations::Int` (for SNIP)
+    *   `window::Int` (for median)
+
+> **Why these inputs?**
+> Baseline correction requires only the intensity vector. The method-specific parameters control the aggressiveness and shape of the baseline estimate.
+>
+> *   **SNIP (Sensitive Nonlinear Iterative Peak clipping)**: Iteratively replaces each point with the minimum of itself and the average of its neighbors. After many iterations, only the baseline remains. SNIP is the default in **MALDIquant** due to its effectiveness.
+> *   **Convex hull**: Constructs the upper convex hull of the spectrum (or lower hull for baseline) and interpolates.
+> *   **Moving median**: Estimates baseline as the local median. Simple but can be biased by dense peaks.
+>
+> **Example**: In a spectrum with a rising baseline after SNIP correction, the baseline is removed, revealing peaks sitting on a flat background.
+
+---
+
+### Normalization
+*   **Function**: `apply_normalization_core`, `tic_normalize`, `pqn_normalize`, `median_normalize`, `rms_normalize`
+*   **Purpose**: Correct for differences in total ion abundance between spectra, making intensities comparable across pixels.
+*   **Inputs required**:
+    *   `y::AbstractVector{<:Real}` (per-spectrum) or a matrix `M` (for PQN).
+    *   `method::Symbol`
+
+> **Why these inputs?**
+> Normalization operates on intensities. **PQN** requires a matrix of spectra (rows = *m/z* bins, columns = spectra) because it computes a reference spectrum across all samples.
+>
+> *   **TIC (Total Ion Current)**: Divides each intensity by the sum of all intensities in the spectrum. A simple algorithm but sensitive to a few very intense peaks (e.g., matrix peaks).
+> *   **Median normalization**: Divides by the median intensity. Robust to outliers but can be unstable if many intensities are zero.
+> *   **RMS normalization**: Divides by root-mean-square, preserving the shape of the spectrum while scaling to unit energy.
+> *   **PQN (Probabilistic Quotient Normalization)**: Calculates a reference spectrum and scales each spectrum by the median of the quotients (intensity/reference). This method is preferred when some peaks vary drastically because it downweights their influence.
+>
+> **Example**: Two spectra from different tissue regions: one has a huge lipid peak, the other has lower overall signal. TIC would suppress the lipid peak in the first spectrum, possibly losing biological information. PQN would scale both spectra more equitably.
+
+---
+
+### Peak Detection
+*   **Functions**: `detect_peaks_profile_core`, `detect_peaks_wavelet_core`, `detect_peaks_centroid_core`
+*   **Purpose**: Identify peaks (signals of interest) in each spectrum.
+*   **Inputs required**:
+    *   *m/z*, intensity vectors.
+    *   Method-specific parameters: `snr_threshold`, `half_window`, `min_peak_prominence`, `merge_peaks_tolerance`, etc.
+
+> **Why these inputs?**
+> Peak detection uses both *m/z* and intensity to locate local maxima and evaluate peak quality. Parameters control sensitivity and filtering. A peak is a local maximum that rises significantly above the local noise. Detection algorithms vary by data mode:
+>
+> *   **Profile mode (local maxima)**: Scans for points that are highest in a sliding window (`half_window`). Then filters by **SNR** (signal-to-noise ratio) and prominence (height relative to surrounding minima). Close peaks are merged if within `merge_peaks_tolerance`.
+> *   **Wavelet-based**: Uses **Continuous Wavelet Transform (CWT)** to identify peaks at multiple scales, making it robust to varying peak widths. Local maxima in the CWT magnitude indicate peak positions. Wavelet is often used for complex spectra with overlapping peaks. CWT-based peak detection is implemented in **MALDIquant** as the default method for profile data.
+> *   **Centroid mode**: Simply filters the existing peak list by SNR, as peaks are already identified.
+>
+> **Example**: A spectrum with a small peak on the shoulder of a large peak: wavelet detection may resolve both, while simple local maxima might miss the shoulder.
+
+---
+
+### Peak Selection (Filtering)
+*   **Function**: `apply_peak_selection`
+*   **Purpose**: Remove peaks that do not meet quality criteria, such as minimum SNR, acceptable width, or good Gaussian shape.
+*   **Inputs required**:
+    *   peaks list (from peak detection).
+    *   Thresholds: `min_snr`, `min_fwhm_ppm`, `max_fwhm_ppm`, `min_shape_r2`.
+
+> **Why these inputs?**
+> These thresholds reflect the analyst's confidence in peak quality. They are typically derived from instrument specifications or empirical data.
+>
+> Not all local maxima are true peaks. Noise spikes, baseline artifacts, or electronic interference can produce spurious peaks. Filtering by **SNR** removes noise. **FWHM** (full width at half maximum) in **ppm** ensures that peaks have physically reasonable widths (e.g., isotopic peaks have predictable spacing). **Shape R²** from a Gaussian fit assesses how well the peak resembles a theoretical peak; poor shape may indicate saturation or interference.
+>
+> **Example**: A peak with SNR < 3 is likely noise and is discarded. A peak with FWHM > 100 *ppm* might be background rather than a specific ion.
+
+---
+
+### Calibration
+*(This step requires that you have a list of Internal Standards inside JuliaMSI preprocessing tab filled in)*
+*   **Function**: `calibrate_spectra_core`, `find_calibration_peaks_core`
+*   **Purpose**: Correct systematic mass errors by aligning the *m/z* axis to known reference masses (internal standards).
+*   **Inputs required**:
+    *   spectra (list of `MutableSpectrum`).
+    *   `internal_standards`: dictionary of theoretical *m/z* values.
+    *   `ppm_tolerance`: matching tolerance.
+    *   `fit_order`: polynomial order for calibration curve.
+
+> **Why these inputs?**
+> Calibration requires a set of known masses that are present in the spectra. The algorithm matches detected peaks to these references, then fits a transformation (e.g., linear) to map measured to theoretical *m/z*.
+>
+> Mass spectrometers drift; this step is made to correct this drift, ensuring that the same *m/z* value corresponds to the same ion across the entire imaging dataset. **Linear or quadratic calibration** is standard. The code currently uses linear interpolation between matched peaks, which is sufficient for most MSI applications where drift is small.
+>
+> **Example**: A reference peak expected at **760.585 Da** (a lipid) is found at **760.590 Da** in a spectrum. Calibration shifts the entire axis so that the peak aligns with the theoretical value.
+
+---
+
+### Peak Alignment
+*   **Function**: `align_peaks_lowess_core`
+*   **Purpose**: Correct residual *m/z* shifts between spectra after calibration, ensuring that the same analyte peak appears at the same *m/z* across all pixels.
+*   **Inputs required**:
+    *   Reference spectrum's peak *m/z* list.
+    *   Target spectrum's peak *m/z* list.
+    *   Parameters: method (`:lowess`, `:linear`, `:ransac`), `tolerance`, `tolerance_unit`, `max_shift_ppm`.
+
+> **Why these inputs?**
+> Alignment operates on peak lists, not full spectra, to be efficient. It finds common peaks between a reference spectrum (e.g., the one with most peaks) and each target, then fits a warping function.
+>
+> Even after calibration, small nonlinear shifts may persist due to sample topography or matrix effects. Alignment brings all spectra onto a common *m/z* scale, which is essential for constructing a feature matrix.
+>
+> **LOWESS (Locally Weighted Scatterplot Smoothing)**: Fits a smooth curve through the matched peak pairs, allowing for nonlinear corrections. It is robust to outliers and does not assume a global polynomial shape. It has been used in **MALDIquant**. **RANSAC (Random Sample Consensus)** is a robust alternative. The code defaults to LOWESS because it is well-suited for smooth, continuous drift.
+>
+> **Example**: Peaks in a target spectrum are systematically shifted by **+0.01 Da** at low *m/z* and **+0.005 Da** at high *m/z*. LOWESS fits a curve that corrects each peak individually, producing aligned *m/z* values.
+
+---
+
+### Peak Binning and Feature Matrix Generation
+*   **Function**: `bin_peaks_core`
+*   **Purpose**: Group peaks from all spectra into common *m/z* bins, producing a final feature matrix.
+*   **Inputs required**:
+    *   spectra (with detected peaks).
+    *   `params::PeakBinning`: method (`:adaptive` or `:uniform`), `tolerance`, `frequency_threshold`, etc.
+
+> **Why these inputs?**
+> Binning requires the list of peaks from each spectrum. The method determines how bins are defined.
+>
+> *   **Adaptive binning**: Clusters peaks based on their *m/z* proximity (using tolerance). Bins are created where peaks are dense, and bins without enough peaks are discarded.
+> *   **Uniform binning**: Divides the total *m/z* range into a fixed number of equally spaced bins. 
+>
+> After alignment, peaks from different spectra still have slightly different *m/z* values due to residual noise. Binning assigns them to discrete bins, creating a matrix suitable for multivariate analysis. The **frequency threshold** removes bins that appear in too few spectra (likely noise).
+>
+> **Example**: Peaks around *m/z* **760.58** from 100 spectra are grouped into one bin. The intensity for each spectrum in that bin is the maximum intensity of peaks assigned to it.
+
+---
+
+## Example Walkthrough: From Raw Spectra to Feature Matrix
+### with Intelligent Parameter Recommendations
+
+To illustrate the entire pipeline, consider a mass spectrometry imaging dataset of a mouse brain section acquired in **profile mode**. The analyst has a list of internal standards for lipids (e.g., known phospholipids at *m/z* **760.585**, **798.540**, **834.528**) to aid in mass calibration.
+
+Before running the full preprocessing, the **JuliaMSI** code executes the intelligent pre-analysis pipeline (`main_precalculation` which is a wrapper for `run_preprocessing_analysis`). This function samples a representative subset of spectra (e.g., 100 pixels) and performs a battery of diagnostic tests to suggest optimal parameters.
+
+#### The pre-analysis yields a report:
+
+*   **Phase 1 – Instrument & Data Characteristics**:
+    *   Acquisition mode: **profile** (continuous waveforms)
+    *   *m/z* axis type: **regular** (constant step of ~0.01 Da)
+    *   Dynamic range: ~4 orders of magnitude
+*   **Phase 2 – Noise & Signal Quality**:
+    *   Estimated noise level (median absolute deviation): 45 counts
+    *   Suggested SNR threshold: **3.0**
+    *   TIC coefficient of variation: **35%** → indicates moderate heterogeneity, so robust normalization (e.g., **PQN**) may be beneficial.
+*   **Phase 3 – Mass Accuracy Analysis** (using the provided lipid standards):
+    *   Mean PPM error: **12.3 ppm**
+    *   Standard deviation: **8.1 ppm**
+    *   Suggested bin tolerance (mean + 3σ): **36.6 ppm** (capped between 10 and 100 ppm).
+*   **Phase 4 – Spatial Region Analysis**:
+    *   *(if region masks were supplied) – skipped in this example.*
+*   **Phase 5 – Peak Characteristics**:
+    *   Mean FWHM: **22.5 ppm** (converted from Δm at *m/z* 500)
+    *   Mean Gaussian R²: **0.85** (peaks are well-shaped)
+    *   Mean peaks per spectrum: **187**
+*   **Phase 6 – Preprocessing Recommendations**:
+    *   The system synthesizes all information into a dictionary of suggested parameters for each step. Parameters that cannot be determined automatically (e.g., frequency threshold for binning, minimum matched peaks) are left as `nothing`; the analyst must supply them based on experimental goals.
+
+> [!NOTE]
+> If you have internal standards, remember to re-run the preprocessing recommendations after you have loaded them. This is only for precalculation and is not a required step for analysts that know about their data.
+
+#### Execution of the Pipeline:
+The analyst can now execute the pipeline, possibly overriding a few parameters (e.g., increasing SNR threshold to 4 for more stringent peak picking).
+
+1.  **Validation**: All spectra pass QC; one empty spectrum is flagged and removed.
+2.  **Intensity transformation (sqrt)**: Stabilizes variance, making low-intensity features more comparable.
+3.  **Baseline correction (SNIP, 100 iterations)**: Removes background using the recommended window of 200 points.
+4.  **Smoothing (Savitzky–Golay, window 9, order 2)**: Reduces noise while preserving peak shape.
+5.  **Peak detection (profile mode)**: Detects peaks with SNR ≥ 3, prominence ≥ 0.002, and uses the recommended half-window of 5 points. Peaks closer than 18 *ppm* are merged.
+6.  **Peak selection**: Filters peaks: keeps only those with SNR ≥ 3, FWHM between 11 and 55 *ppm*, and shape R² ≥ 0.68.
+7.  **Calibration**: Using the internal standards, each spectrum’s *m/z* axis is linearly corrected based on matched peaks within 36.6 *ppm* tolerance.
+8.  **Peak alignment (LOWESS)**: Aligns all spectra to a reference spectrum (the one with most peaks) with a span of 0.6, using the same 36.6 *ppm* tolerance. This corrects residual nonlinear drift.
+9.  **Normalization (RMS)**: Divides each spectrum by its root-mean-square intensity, chosen because TIC variation was high.
+10. **Peak binning (adaptive)**: Groups peaks within 36.6 *ppm* into bins. Bins containing fewer than 3 peaks or appearing in <20% of spectra are discarded. The final bin centers are intensity-weighted averages.
+11. **Feature matrix**: A **999 × 425 matrix** is produced, where rows are pixels and columns are *m/z* bins. This matrix is ready for PCA, segmentation, or statistical testing.
+
+#### How the Analyst Uses the Recommendations:
+*   The suggested values are data-driven, but the analyst can override them based on domain knowledge. Always verify; experimenting is encouraged as data integrity is kept.
+*   Parameters like `frequency_threshold` in peak selection and binning require the analyst’s judgment. A threshold of **0.2** means that a bin must contain a peak in at least 20% of all pixels to be retained. This reduces noise but may discard rare but real signals.
+*   This is an iterative refinement; the analyst may rerun the analysis with adjusted parameters (e.g., a lower bin tolerance).
+*   JuliaMSI aims for a tailored preprocessing workflow that respects both the instrument’s characteristics and the biological question.
+
+---
+
+## Extra Tips for the Analyst
+
+The preprocessing pipeline is powerful, but its success ultimately depends on the analyst’s informed choices. Here are practical tips to guide you through the process, especially when working with internal standards and interpreting intermediate results.
+
+> [!WARNING]
+> This is a really resource-intensive pipeline dependent on the sample, so run your precautions.
+
+### Isotopic and Charge State Matters
+When providing a dictionary of reference masses for calibration, ensure that the theoretical *m/z* values correspond to the same charge state and isotopic composition as the peaks you expect to see in your data.
+*   **Example**: If your instrument detects lipids mainly as **[M+H]⁺** ions, the reference mass must be the monoisotopic mass plus a proton. Using the neutral mass will introduce a systematic offset of **1.0078 Da**, leading to poor calibration. (Make the same adjustment for negative ion mode).
+*   If isotopic peaks are prominent, you may want to include multiple isotopes (e.g., **M**, **M+1**, **M+2**) as separate references, but be aware that their relative intensities vary. The calibration algorithm will match the closest detected peak. This is common for MSI preprocessing.
+*   The pre-analysis suggests a bin tolerance based on observed mass error. A tolerance that is too large may match the wrong peak (e.g., an isobaric interference); a tolerance that is too small may miss valid matches.
+
+### Always Visualize the Average Spectrum First
+Before running any preprocessing, generate a sum or average spectrum. This single plot tells you:
+1.  **Baseline shape**: If it's flat, rising, or undulating, it can guide the baseline correction method (**SNIP** works well for most shapes; **convex hull** is faster but may fail on curvy baselines).
+2.  **Noise level**: By zooming in.
+3.  ***m/z* Scatter**: How scattered are the spectra *m/z*.
+4.  **Peak density & Intensity range**: Influence the peak detection thresholds (SNR, prominence) and binning strategy.
+
+*   If the average spectrum shows a **rising baseline** from low to high *m/z*, SNIP with ~100 iterations will handle it well. If extremely wavy, increase iterations or try a different method.
+*   If running from the console (`run_preprocessing.jl`), the output of `run_preprocessing_analysis` provides:
+    *   **TIC coefficient of variation (CV)**: A CV > 30% suggests substantial pixel-to-pixel variation, favoring **RMS** or **PQM** over TIC.
+    *   **Mean PPM error**: If >20 *ppm*, calibration is essential. If <5 *ppm* and drift is linear, you might skip alignment.
+    *   **FWHM and Gaussian R²**: Indicate peak quality. If R² < 0.6, peaks may be poorly shaped; consider increasing the `min_peak_shape_r2` threshold.
+*   **Acquisition mode**: For **centroid data**, never apply smoothing or baseline correction because those steps assume continuous waveforms.
+    *   The pipeline will set `:method = :centroid` in peak picking and use permissive filters.
+    *   Use a low SNR threshold (e.g., 2.0).
+    *   Disable shape-based filtering (`min_peak_shape_r2 = 0.0`).
+    *   Centroid data do not retain peak width information; set `min_fwhm_ppm = 0.0` and `max_fwhm_ppm = 500.0` to avoid discarding real peaks.
+    *   Adaptive binning still works; tolerance should be based on expected mass accuracy.
+
+### Preprocessing Is Not a One-Shot Process
+The recommended parameters are a starting point, not a final answer.
+*   **Reproducibility**: Keep a record of all parameters used. The `params` dictionary can be saved as a **JSON**. This is essential for publications.
+*   **Biological Context**: If looking for rare metabolites appearing in few pixels, lower the `frequency_threshold` in binning. If interested only in high-abundance lipids, increase the SNR threshold.
+*   **Modularity**: The pipeline is modular; you can always rerun parts of it with adjusted parameters without starting from scratch.
+
+---
+*Any revision to the code, bug, error, report to julian.sierrag@icloud.com, if you we're to cite this file, remember i provide no citation and its based on internet knowledge of the web about these algorithms, so do your own research and cite properly. to cite JuliaMSI platform, please cite:*
+
+José Julián Sierra-Álvarez, Martín Orlando Camargo-Escalante, Carlos Daniel Sierra-Álvarez, Carmelo Hernández-Caricio, Juan Francisco Moreno-Luna, Isabel Buendía-Corona, Robert Winkler,
+JuliaMSI: A high-performance graphical platform for mass spectrometry imaging data analysis,
+Analytica Chimica Acta,
+Volume 1377,
+2025,
+344613,
+ISSN 0003-2670,
+https://doi.org/10.1016/j.aca.2025.344613

Project.toml

@@ -1,4 +1,5 @@
 name = "MSI_src"
+uuid = "8f395f85-3b9c-4f7f-bf7e-12345678abcd"
 authors = ["JJSA"]
 version = "0.1.0"
 
@@ -35,6 +36,7 @@ Loess = "4345ca2d-374a-55d4-8d30-97f9976e7612"
 Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
 NativeFileDialog = "e1fe445b-aa65-4df4-81c1-2041507f0fd4"
 NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85"
+PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d"
 Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a"
 PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
@@ -42,9 +44,11 @@ ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 SavitzkyGolay = "c4bf5708-b6a6-4fbe-bcd0-6850ed671584"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 StipplePlotly = "ec984513-233d-481d-95b0-a3b58b97af2b"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
@@ -52,8 +56,10 @@ Accessors = "0.1"
 Base64 = "1.11"
 CSV = "0.10"
 CairoMakie = "0.13"
+CodecBase = "0.3"
 ColorSchemes = "3.30"
 Colors = "0.12"
+ContinuousWavelets = "1.1"
 DataFrames = "1.7"
 Dates = "1.11"
 FileIO = "1.17"
@@ -64,11 +70,13 @@ HistogramThresholding = "0.3"
 ImageBinarization = "0.3"
 ImageComponentAnalysis = "0.2"
 ImageContrastAdjustment = "0.3"
-ImageCore = "0.10.5"
+ImageCore = "0.10"
 ImageFiltering = "0.7"
 ImageMorphology = "0.4"
 ImageSegmentation = "1.9"
 Images = "0.26"
+Interpolations = "0.15"
+JLD2 = "0.6"
 JSON = "0.21"
 Libz = "1.0"
 LinearAlgebra = "1.11"
@@ -76,13 +84,22 @@ Loess = "0.6"
 Mmap = "1.11"
 NativeFileDialog = "0.2"
 NaturalSort = "1.0"
+Parameters = "0.12"
 PlotlyBase = "0.8"
 Printf = "1.11"
 ProgressMeter = "1.11"
 SavitzkyGolay = "0.9"
 Serialization = "1.11"
+Setfield = "1.1"
 Statistics = "1.11"
 StatsBase = "0.34"
 StipplePlotly = "0.13"
+Test = "1.11.0"
 UUIDs = "1.11"
-julia = "1.11"
+julia = "1.11, 1.12"
+
+[extras]
+Test = "8dfed614-e60c-5e00-aba5-a33c2d43236e"
+
+[targets]
+test = ["Test"]

README.md

@@ -3,9 +3,10 @@
 https://codeberg.org/LabABI/JuliaMSI
 
 ## Local Installation
-1. Make sure you have Julia installed and at least Julia version 1.6, If not, follow the next installation guide which contains the latest files and instructions for [juliaup](https://github.com/JuliaLang/juliaup) (recommended) alternatively, you can follow the official Julia installation instructions at [julia](https://julialang.org/downloads/platform/).
+1. Make sure you have Julia installed and at least Julia version 1.6 but not higher than 1.11, If not, follow the next installation guide which contains the latest files and instructions for [juliaup](https://github.com/JuliaLang/juliaup) (recommended) alternatively, you can follow the official Julia installation instructions at [julia](https://julialang.org/downloads/platform/).
 2. Download and decompress the following repository (You can download the repository from [download](https://codeberg.org/LabABI/JuliaMSI/archive/main.zip) or locate the option manually at the top right of the page, in the box with [•••] (More operations). For a more advanced method using Git, use this link:  https://codeberg.org/LabABI/JuliaMSI.git).
 3. Ensure that you have decompressed the ZIP file and know the working directory where it is located (the folder on your computer where you saved the JuliaMSI files).
+4. Julia Version 1.11 is required. You might need to downgrade your whole Julia installation to support it.
 
 ## Load User Interface
 1. Set the working directory to where JuliaMSI is located (the specific location of the folder on your computer where you decompressed the ZIP) in your terminal.<br>
@@ -21,7 +22,7 @@ https://codeberg.org/LabABI/JuliaMSI
    Example of a correct route:<br>
    ~/Downloads/JuliaMSI-main/juliamsi
 2. Without entering the Julia environment, launch the project in your terminal with the following command (which works for all operating systems):
-   ```
+   ```bash
    julia --threads auto --project=. start_MSI_GUI.jl
    ```
 3. After the script has finished loading, you can open a [page](http://127.0.0.1:1481/) in your browser with the web app running.
@@ -33,6 +34,47 @@ Minimum system requirements: 4 core processor, 8 GB RAM<br>
 
 JuliaMSI is a Graphical User Interface for a library of MSI tools in Julia: https://github.com/CINVESTAV-LABI/julia_mzML_imzML
 
+## Build the System Image (One-time)
+Alternatively, you can generate custom pre-compiled `.so` / `.dll` system images by running the build script in your directory. This bakes the heavy mass-spectrometry kernels and UI dependencies into a single machine-code binary for ultra-fast startup times.
+
+### 1. Build the GUI Sysimage (for full App usage)
+```bash
+julia --project=. build_sysimage.jl
+```
+This may take 5–15 minutes. The resulting `sys_msi_gui.so` (or `.dll` on Windows) file will be around **300MB–600MB** because it contains the pre-compiled machine code for your entire graphical environment.
+*Note: A sysimage built on Linux (.so) will not work on Windows. You must run the build script once on each target operating system.*
+
+Once `sys_msi_gui.so` is created in your directory, adapt your launch command:
+```bash
+julia --project=. -e 'using Pkg; Pkg.precompile()'
+# Adjust the .so extension to .dll if on Windows or .dylib if on macOS
+julia --threads auto --project=. --sysimage sys_msi_gui.so start_MSI_GUI.jl
+```
+
+### 2. Build the Headless Sysimage (for Scripts & Data Scientists)
+If you only want to run scripts or the core `MSI_src` engine without the overhead of the Genie web server and Plotly, you can build a stripped-down, high-speed system image:
+
+```bash
+julia --project=. build_sysimage.jl --headless
+```
+
+Launch your automated tests or custom processing scripts using the headless image to bypass virtually all compilation overhead:
+```bash
+julia --threads auto --project=. --sysimage sys_msi_headless.so test/test_streaming_pipeline.jl
+```
+
+### 3. Docker Option for High-Speed Processing
+For maximum portability and execution speed without clutter, we provide a `Dockerfile.headless`. This option creates a minimal, ultra-fast container that completely omits `app.jl` and the web interface. 
+
+It automatically builds and bundles the headless sysimage, giving researchers an instant-start engine ready for heavy-duty `.imzML` batch pipelines with zero JIT latency.
+
+To build and run the Docker image:
+```bash
+docker build -t juliamsi-headless -f Dockerfile.headless .
+# Run a script mapped from your local data folder:
+docker run -it -v /my/local/data:/data juliamsi-headless julia -J sys_msi_headless.so /data/my_processing_script.jl
+```
+
 ## License
 
 JuliaMSI is published under the terms of the MIT License.
@@ -41,3 +83,9 @@ JuliaMSI is published under the terms of the MIT License.
 
 Robert Winkler. (2023). mzML mass spectrometry and imzML mass spectrometry imaging test data [Data set]. 
 Zenodo. <https://doi.org/10.5281/zenodo.10084132>
+
+## User Guides
+
+For a general guide on how to use the JuliaMSI platform, please refer to the [JuliaMSI Main Guide](JuliaMSI_MainGuide.md).
+
+For a more detailed guide on the preprocessing pipeline, please refer to the [JuliaMSI Preprocessing Guide](JuliaMSI_Preprocessing.md).

build_sysimage.jl

@@ -0,0 +1,49 @@
+# build_sysimage.jl
+using PackageCompiler
+
+headless = "--headless" in ARGS
+
+println("--- Starting Custom System Image Build ---")
+println("Mode: ", headless ? "Headless (Scripting/Docker)" : "Full GUI (Genie App)")
+println("This process can take 5-15 minutes.")
+
+ext = Sys.iswindows() ? ".dll" : Sys.isapple() ? ".dylib" : ".so"
+sysimage_name = headless ? "sys_msi_headless" * ext : "sys_msi_gui" * ext
+sysimage_path = joinpath(@__DIR__, sysimage_name)
+
+packages_to_include = Symbol[
+    :MSI_src, # <--- Bake our core library
+    :DataFrames,
+    :SparseArrays,
+    :Mmap,
+    :Libz,
+    :Serialization,
+    :Printf,
+    :JSON
+]
+
+if !headless
+    append!(packages_to_include, [
+        :Genie,
+        :GenieFramework,
+        :PlotlyBase
+    ])
+end
+
+create_sysimage(
+    packages_to_include,
+    sysimage_path = sysimage_path,
+    precompile_execution_file = "precompile_script.jl",
+    incremental = true,
+    filter_stdlibs = false
+)
+
+println("--- System Image Build Successful! ---")
+println("Created: $(sysimage_path)")
+if headless
+    println("To use in a headles script:")
+    println("julia --threads auto --project=. --sysimage $(sysimage_name) my_script.jl")
+else
+    println("To start Julia with the GUI image, use:")
+    println("julia --threads auto --project=. --sysimage $(sysimage_name) start_MSI_GUI.jl")
+end

config/env/prod.jl

@@ -0,0 +1,6 @@
+# config/env/prod.jl
+# This file is loaded when GENIE_ENV is set to "prod"
+
+# Define PLUGINS_WITH_ASSETS to prevent UndefVarError in production mode
+# It's typically an empty vector if no specific plugins with assets are tracked.
+const PLUGINS_WITH_ASSETS = []

julia_imzML_visual.jl

@@ -1,6 +1,5 @@
 # julia_imzML_visual.jl
 
-const REGISTRY_LOCK = ReentrantLock()
 
 """
     increment_image(current_image, image_list)
@@ -531,7 +530,13 @@ function meanSpectrumPlot(data::MSIData, dataset_name::String=""; mask_path::Uni
             showgrid=true,
             tickformat=".3g"
         ),
-        margin=attr(l=0, r=0, t=120, b=0, pad=0)
+        margin=attr(l=0, r=0, t=120, b=0, pad=0),
+        legend=attr(
+            x=1.0, 
+            y=1.0, 
+            xanchor="right", 
+            yanchor="top"
+        )
     )
 
     # Use the new, efficient function from the backend
@@ -544,12 +549,13 @@ function meanSpectrumPlot(data::MSIData, dataset_name::String=""; mask_path::Uni
         layout.title.text = "Empty " * layout.title.text
     else
         df = data.spectrum_stats_df
-        profile_count = 0
+        plot_as_lines = false # Default to stem for safety if no mode info
-        if df !== nothing && hasproperty(df, :Mode)
+        if df !== nothing && hasproperty(df, :Mode) && !isempty(df.Mode)
             profile_count = count(==(MSI_src.PROFILE), df.Mode)
+            plot_as_lines = profile_count > length(df.Mode) / 2
         end
 
-        if profile_count > 0
+        if plot_as_lines
             trace = PlotlyBase.scatter(x=xSpectraMz, y=ySpectraMz, mode="lines", marker=attr(size=1, color="blue", opacity=0.5), name="Average", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
         else
             trace = PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz, marker=attr(size=1, color="blue", opacity=0.5), name="Average", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
@@ -582,22 +588,25 @@ Generates a plot for the spectrum at a specific coordinate (for imaging data) or
 """
 function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int, imgHeight::Int, dataset_name::String=""; mask_path::Union{String, Nothing}=nothing)
     local mz::AbstractVector, intensity::AbstractVector
-    local plot_title::String
+    local spectrum_id::Int = -1 # Initialize spectrum_id
-    local spectrum_mode = MSI_src.CENTROID # Default to centroid
+    mz, intensity = Float64[], Float64[]
+    base_title = ""
+    spectrum_mode = MSI_src.PROFILE
 
-    is_imaging = data.source isa ImzMLSource
+    if data.source isa ImzMLSource
-
-    if is_imaging
-        x = clamp(xCoord, 1, imgWidth)
-        y = clamp(yCoord, 1, imgHeight)
-
-        # Get spectrum mode
-        if data.spectrum_stats_df !== nothing && hasproperty(data.spectrum_stats_df, :Mode)
         w, h = data.image_dims
-            if y > 0 && x > 0 && y <= h && x <= w
+        x = clamp(xCoord, 1, w)
-                idx = (y - 1) * w + x
+        y = clamp(yCoord, 1, h)
-                if idx <= length(data.spectrum_stats_df.Mode)
+        spectrum_id = (y - 1) * w + x # Calculate spectrum_id for imaging data
-                    spectrum_mode = data.spectrum_stats_df.Mode[idx]
+
+        # Check if spectrum stats are available and retrieve the mode
+        if data.spectrum_stats_df !== nothing && hasproperty(data.spectrum_stats_df, :Mode)
+            if spectrum_id <= length(data.spectrum_stats_df.Mode)
+                try
+                    spectrum_mode = data.spectrum_stats_df.Mode[spectrum_id]
+                catch e
+                    @warn "Could not retrieve spectrum mode for index $spectrum_id. Defaulting to PROFILE."
+                    spectrum_mode = MSI_src.PROFILE
                 end
             end
         end
@@ -625,6 +634,7 @@ function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int,
     else
         # For non-imaging data, treat xCoord as the spectrum index
         index = clamp(xCoord, 1, length(data.spectra_metadata))
+        spectrum_id = index # Assign index to spectrum_id for non-imaging data
 
         if data.spectrum_stats_df !== nothing && hasproperty(data.spectrum_stats_df, :Mode)
             if index <= length(data.spectrum_stats_df.Mode)
@@ -660,7 +670,13 @@ function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int,
             showgrid=true,
             tickformat=".3g"
         ),
-        margin=attr(l=0, r=0, t=120, b=0, pad=0)
+        margin=attr(l=0, r=0, t=120, b=0, pad=0),
+        legend=attr(
+            x=1.0, 
+            y=1.0, 
+            xanchor="right", 
+            yanchor="top"
+        )
     )
 
     # Downsample for plotting performance
@@ -675,7 +691,98 @@ function xySpectrumPlot(data::MSIData, xCoord::Int, yCoord::Int, imgWidth::Int,
     plotdata = [trace]
     plotlayout = layout
 
-    return plotdata, plotlayout, mz, intensity
+    return plotdata, plotlayout, mz, intensity, spectrum_id
+end
+
+"""
+    nSpectrumPlot(data::MSIData, id::Int, dataset_name::String=""; mask_path::Union{String, Nothing}=nothing)
+
+Generates a plot for a spectrum specified by its linear `id` for any type of MSIData.
+
+# Arguments
+- `data`: The `MSIData` object.
+- `id`: The linear index (ID) of the spectrum to plot.
+- `dataset_name`: Optional name of the dataset for the plot title.
+- `mask_path`: Optional path to a mask file (currently not used for plotting by ID, but kept for signature consistency).
+
+# Returns
+- `plotdata`: A vector containing the Plotly trace.
+- `plotlayout`: The Plotly layout for the plot.
+- `mz`: The m/z values of the spectrum.
+- `intensity`: The intensity values of the spectrum.
+- `spectrum_id`: The linear index of the spectrum (same as input `id`).
+"""
+function nSpectrumPlot(data::MSIData, id::Int, dataset_name::String=""; mask_path::Union{String, Nothing}=nothing)
+    local mz::AbstractVector, intensity::AbstractVector
+    local plot_title::String
+    local spectrum_mode = MSI_src.PROFILE # Default to profile
+    local spectrum_id::Int = id # Spectrum ID is the input id
+
+    # Validate ID
+    if id < 1 || id > length(data.spectra_metadata)
+        @warn "Spectrum ID $id is out of bounds."
+        mz, intensity = Float64[], Float64[]
+        base_title = "Spectrum ID $id (Out of Bounds)"
+        spectrum_id = 0 # Indicate invalid spectrum ID
+    else
+        # Get spectrum mode
+        if data.spectrum_stats_df !== nothing && hasproperty(data.spectrum_stats_df, :Mode)
+            if id <= length(data.spectrum_stats_df.Mode)
+                spectrum_mode = data.spectrum_stats_df.Mode[id]
+            end
+        end
+
+        # Retrieve spectrum data
+        process_spectrum(data, id) do recieved_mz, recieved_intensity
+            mz = recieved_mz
+            intensity = recieved_intensity
+        end
+        base_title = "Spectrum #$id"
+    end
+
+    plot_title = isempty(dataset_name) ? base_title : "$base_title for: $dataset_name"
+
+    layout = PlotlyBase.Layout(
+        title=PlotlyBase.attr(
+            text=plot_title,
+            font=PlotlyBase.attr(
+                family="Roboto, Lato, sans-serif",
+                size=18,
+                color="black"
+            )
+        ),
+        hovermode="closest",
+        xaxis=PlotlyBase.attr(
+            title="<i>m/z</i>",
+            showgrid=true
+        ),
+        yaxis=PlotlyBase.attr(
+            title="Intensity",
+            showgrid=true,
+            tickformat=".3g"
+        ),
+        margin=attr(l=0, r=0, t=120, b=0, pad=0),
+        legend=attr(
+            x=1.0, 
+            y=1.0, 
+            xanchor="right", 
+            yanchor="top"
+        )
+    )
+
+    # Downsample for plotting performance
+    mz_down, int_down = MSI_src.downsample_spectrum(mz, intensity)
+
+    trace = if spectrum_mode == MSI_src.CENTROID
+        PlotlyBase.stem(x=mz_down, y=int_down, marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
+    else
+        PlotlyBase.scatter(x=mz_down, y=int_down, mode="lines", marker=attr(size=1, color="blue", opacity=0.5), name="Spectrum", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
+    end
+
+    plotdata = [trace]
+    plotlayout = layout
+
+    return plotdata, plotlayout, mz, intensity, spectrum_id
 end
 
 """
@@ -722,7 +829,13 @@ function sumSpectrumPlot(data::MSIData, dataset_name::String=""; mask_path::Unio
             showgrid=true,
             tickformat=".3g"
         ),
-        margin=attr(l=0, r=0, t=120, b=0, pad=0)
+        margin=attr(l=0, r=0, t=120, b=0, pad=0),
+        legend=attr(
+            x=1.0, 
+            y=1.0, 
+            xanchor="right", 
+            yanchor="top"
+        )
     )
 
     # Use the get_total_spectrum function from the backend
@@ -735,12 +848,13 @@ function sumSpectrumPlot(data::MSIData, dataset_name::String=""; mask_path::Unio
         layout.title.text = "Empty " * layout.title.text
     else
         df = data.spectrum_stats_df
-        profile_count = 0
+        plot_as_lines = false # Default to stem for safety
-        if df !== nothing && hasproperty(df, :Mode)
+        if df !== nothing && hasproperty(df, :Mode) && !isempty(df.Mode)
             profile_count = count(==(MSI_src.PROFILE), df.Mode)
+            plot_as_lines = profile_count > length(df.Mode) / 2
         end
 
-        if profile_count > 0
+        if plot_as_lines
             trace = PlotlyBase.scatter(x=xSpectraMz, y=ySpectraMz, mode="lines", marker=attr(size=1, color="blue", opacity=0.5), name="Total", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
         else
             trace = PlotlyBase.stem(x=xSpectraMz, y=ySpectraMz, marker=attr(size=1, color="blue", opacity=0.5), name="Total", hoverinfo="x", hovertemplate="<b>m/z</b>: %{x:.4f}<extra></extra>")
@@ -789,10 +903,22 @@ function warmup_init()
     end
 end
 
+"""
+    clean_registry_path(registry_path::String)
+
+Sanitizes the file path to prevent OS/file system issues, particularly on Windows,
+by standardizing separators and removing stripping whitespace.
+"""
+function clean_registry_path(registry_path::String)
+    # Standardize to forward slashes internally for uniformity, then normpath formats for OS
+    clean_path = normpath(strip(replace(registry_path, "\\" => "/")))
+    return abspath(clean_path)
+end
+
 """
     load_registry(registry_path)
 
-Loads the dataset registry from a JSON file.
+Loads the dataset registry from a JSON file, with retries for robustness on Windows.
 
 # Arguments
 - `registry_path`: Path to the `registry.json` file.
@@ -800,77 +926,114 @@ Loads the dataset registry from a JSON file.
 # Returns
 - A dictionary containing the registry data. Returns an empty dictionary if the file doesn't exist or fails to parse.
 """
-function load_registry(registry_path)
+function load_registry(registry_path::String)
     return lock(REGISTRY_LOCK) do
-        if isfile(registry_path)
+        clean_path = clean_registry_path(registry_path)
+        if isfile(clean_path)
+            for attempt in 1:5
                 try
-                JSON.parsefile(registry_path, dicttype=Dict{String,Any})
+                    return JSON.parsefile(clean_path, dicttype=Dict{String,Any})
                 catch e
-                @error "Failed to parse registry.json: $e"
+                    if attempt == 5
-                Dict{String,Any}()
+                        @error "Final attempt failed to parse registry.json: $(sprint(showerror, e))"
-            end
+                        return Dict{String,Any}()
                     else
-            Dict{String,Any}()
+                        @warn "Attempt $attempt to read registry.json failed, retrying in 0.2s... Error: $(sprint(showerror, e))"
+                        sleep(0.2)
                     end
                 end
+            end
+        end
+        return Dict{String,Any}()
+    end
 end
 
 """
-    extract_metadata(msi_data::MSIData, source_path::String)
+    extract_metadata(loaded_data::MSIData, full_route::String)
 
-Extracts key metadata from an `MSIData` object for display.
+Extracts key metadata from an MSIData object and file path for UI display.
-
+Returns a dictionary with a "summary" key containing parameter-value pairs.
-# Arguments
-- `msi_data`: The `MSIData` object.
-- `source_path`: The path to the source data file.
-
-# Returns
-- A dictionary containing summary statistics and metadata.
 """
-function extract_metadata(msi_data::MSIData, source_path::String)
+function extract_metadata(loaded_data::MSIData, full_route::String)
-    df = msi_data.spectrum_stats_df
+    stats = []
-    if df === nothing
+    
-        # This can happen if precompute_analytics hasn't been run
+    # Basic File Info
-        # We can still return basic info
+    push!(stats, Dict("parameter" => "Filename", "value" => basename(full_route)))
-        return Dict(
+    push!(stats, Dict("parameter" => "Source Path", "value" => full_route))
-            "summary" => [
+    
-                Dict("parameter" => "File Name", "value" => basename(source_path)),
+    # Image Dimensions
-                Dict("parameter" => "Number of Spectra", "value" => length(msi_data.spectra_metadata)),
+    w, h = loaded_data.image_dims
-                Dict("parameter" => "Image Dimensions", "value" => "$(msi_data.image_dims[1]) x $(msi_data.image_dims[2])"),
+    if w > 0 && h > 0
-            ],
+        push!(stats, Dict("parameter" => "Image Dimensions", "value" => "$w x $h"))
-            "global_min_mz" => nothing,
+    else
-            "global_max_mz" => nothing
+        push!(stats, Dict("parameter" => "Image Dimensions", "value" => "N/A (Non-imaging)"))
-        )
     end
     
-    summary_stats = [
+    # Spectrum Counts
-        Dict("parameter" => "File Name", "value" => basename(source_path)),
+    num_spectra = length(loaded_data.spectra_metadata)
-        Dict("parameter" => "Number of Spectra", "value" => length(msi_data.spectra_metadata)),
+    push!(stats, Dict("parameter" => "Total Spectra", "value" => string(num_spectra)))
-        Dict("parameter" => "Image Dimensions", "value" => "$(msi_data.image_dims[1]) x $(msi_data.image_dims[2])"),
-        Dict("parameter" => "Global Min m/z", "value" => @sprintf("%.4f", Threads.atomic_add!(msi_data.global_min_mz, 0.0))),
-        Dict("parameter" => "Global Max m/z", "value" => @sprintf("%.4f", Threads.atomic_add!(msi_data.global_max_mz, 0.0))),
-        Dict("parameter" => "Mean TIC", "value" => @sprintf("%.2e", mean(df.TIC))),
-        Dict("parameter" => "Mean BPI", "value" => @sprintf("%.2e", mean(df.BPI))),
-        Dict("parameter" => "Mean # Points", "value" => @sprintf("%.1f", mean(df.NumPoints))),
-    ]
     
-    if hasproperty(df, :Mode)
+    # m/z Range
-        centroid_count = count(==(MSI_src.CENTROID), df.Mode)
+    min_mz, max_mz = get_global_mz_range(loaded_data)
-        profile_count = count(==(MSI_src.PROFILE), df.Mode)
+    if isfinite(min_mz) && isfinite(max_mz)
-        unknown_count = count(==(MSI_src.UNKNOWN), df.Mode)
+        push!(stats, Dict("parameter" => "Global m/z Range", "value" => "$(round(min_mz, digits=4)) - $(round(max_mz, digits=4))"))
+    else
+        push!(stats, Dict("parameter" => "Global m/z Range", "value" => "Unknown"))
+    end
     
-        push!(summary_stats, Dict("parameter" => "Centroid Spectra", "value" => string(centroid_count)))
+    # Instrument Metadata
-        push!(summary_stats, Dict("parameter" => "Profile Spectra", "value" => string(profile_count)))
+    if loaded_data.instrument_metadata !== nothing
-        if unknown_count > 0
+        inst = loaded_data.instrument_metadata
-            push!(summary_stats, Dict("parameter" => "Unknown Mode Spectra", "value" => string(unknown_count)))
+        push!(stats, Dict("parameter" => "Instrument Model", "value" => inst.instrument_model))
+        push!(stats, Dict("parameter" => "Polarity", "value" => titlecase(string(inst.polarity))))
+        push!(stats, Dict("parameter" => "Acquisition Mode", "value" => titlecase(string(inst.acquisition_mode))))
+        
+        if inst.resolution !== nothing
+            push!(stats, Dict("parameter" => "Instrument Resolution", "value" => string(inst.resolution)))
         end
     end
     
-    return Dict(
+    return Dict("summary" => stats)
-        "summary" => summary_stats,
+end
-        "global_min_mz" => msi_data.global_min_mz,
+
-        "global_max_mz" => msi_data.global_max_mz
+"""
-    )
+    save_registry(registry_path::String, registry_data::Dict)
+
+Saves the dataset registry to a JSON file atomically to prevent data loss 
+and sharing violations on Windows. Writes to a `.tmp` file first.
+"""
+function save_registry(registry_path::String, registry_data)
+    lock(REGISTRY_LOCK) do
+        clean_path = clean_registry_path(registry_path)
+        dir = dirname(clean_path)
+        if !isdir(dir)
+            mkpath(dir)
+        end
+
+        temp_path = clean_path * ".tmp"
+
+        for attempt in 1:6
+            try
+                open(temp_path, "w") do f
+                    JSON.print(f, registry_data, 4)
+                end
+                
+                # Replace final file with temp atomically. Force=true maps to ReplaceFile/MoveFileEx
+                mv(temp_path, clean_path, force=true)
+                return true
+            catch e
+                if attempt == 6
+                    @error "Final attempt failed to write to registry.json: $(sprint(showerror, e))"
+                    isfile(temp_path) && rm(temp_path, force=true)
+                    return false
+                else
+                    @warn "Attempt $attempt to write to registry.json failed (Error: $(sprint(showerror, e))), retrying in 0.2s..."
+                    sleep(0.2)
+                end
+            end
+        end
+        return false
+    end
 end
 
 """
@@ -885,44 +1048,22 @@ Adds or updates an entry in the dataset registry JSON file.
 - `metadata`: Optional dictionary of metadata to store.
 - `is_imzML`: Boolean indicating if the source is an imzML file.
 """
-function update_registry(registry_path, dataset_name, source_path, metadata=nothing, is_imzML=false)
+function update_registry(registry_path::String, dataset_name::String, source_path::String, metadata=nothing, is_imzML=false)
     lock(REGISTRY_LOCK) do
-        registry = if isfile(registry_path)
+        registry = load_registry(registry_path)
-            try
-                JSON.parsefile(registry_path, dicttype=Dict{String,Any})
-            catch e
-                @error "Failed to parse registry.json while updating: $e"
-                Dict{String,Any}() # Start with empty if parsing fails
-            end
-        else
-            Dict{String,Any}()
-        end
 
-        # Get existing entry if it exists, otherwise create new one
         existing_entry = get(registry, dataset_name, Dict{String,Any}())
-        
-        # Start with existing data and update only the basic fields
         entry = copy(existing_entry)
         entry["source_path"] = source_path
         entry["processed_date"] = string(now())
         entry["is_imzML"] = is_imzML
         
-        # Only update metadata if provided
         if metadata !== nothing
             entry["metadata"] = metadata
         end
         
-        # Note: has_mask and mask_path are preserved from existing_entry if they exist
-        
         registry[dataset_name] = entry
-
+        save_registry(registry_path, registry)
-        try
-            open(registry_path, "w") do f
-                JSON.print(f, registry, 4)
-            end
-        catch e
-            @error "Failed to write to registry.json: $e"
-        end
     end
 end
 
@@ -990,23 +1131,24 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov
         # --- Extract metadata ---
         metadata = extract_metadata(local_msi_data, file_path)
 
-        # --- Save Slices ---
+        # --- Save Slices (Parallelized) ---
         mkpath(output_dir)
         
-        for (mass_idx, mass) in enumerate(masses)
+        progress_lock = ReentrantLock()
-            progress_message_ref = "File $(params.fileIdx)/$(params.nFiles): Saving slice for m/z=$mass"
         
+        # Parallelize over the requested masses to fully utilize multi-core CPUs
+        Threads.@threads for mass in masses
             slice = slice_dict[mass]
             text_nmass = replace(string(mass), "." => "_")
             bitmap_filename = params.triqE ? "TrIQ_$(text_nmass).bmp" : "MSI_$(text_nmass).bmp"
             colorbar_filename = params.triqE ? "colorbar_TrIQ_$(text_nmass).png" : "colorbar_MSI_$(text_nmass).png"
 
+            local sliceQuant, bounds
             if all(iszero, slice)
                 sliceQuant = zeros(UInt8, size(slice))
                 bounds = (0.0, 1.0)  # Default bounds for empty slice
-                @warn "No intensity data for m/z = $mass in $(dataset_name)"
             else
-                # Get both quantized data AND bounds in one call
+                # TrIQ and quantization are computationally intensive and now run in parallel
                 if params.triqE
                     sliceQuant, bounds = TrIQ(slice, params.colorL, params.triqP, mask_matrix=mask_matrix_for_triq)
                 else
@@ -1015,17 +1157,23 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov
                 
                 if params.medianF
                     sliceQuant = round.(UInt8, median_filter(sliceQuant))
-                    # Note: bounds remain the same after median filter
                 end
             end
 
+            # Disk I/O (Saving BMP)
             save_bitmap(joinpath(output_dir, bitmap_filename), sliceQuant, ViridisPalette)
+            
             if !all(iszero, slice)
-                # Now pass the precomputed bounds to colorbar generation
+                # CairoMakie colorbar generation is now also parallelized
                 generate_colorbar_image(slice, params.colorL, joinpath(output_dir, colorbar_filename), 
                                       bounds; use_triq=params.triqE, triq_prob=params.triqP, mask_path=mask_path)
             end
+            
+            lock(progress_lock) do
+                progress_message_ref = "File $(params.fileIdx)/$(params.nFiles): Saved slice for m/z=$mass"
             end
+        end
+
 
         is_imzML = local_msi_data.source isa ImzMLSource
         update_registry(params.registry, dataset_name, file_path, metadata, is_imzML)
@@ -1046,24 +1194,14 @@ function process_file_safely(file_path, masses, params, progress_message_ref, ov
     end
 end
 
-function update_registry_mask_fields(registry_path, dataset_name, has_mask, mask_path)
+function update_registry_mask_fields(registry_path::String, dataset_name::String, has_mask::Bool, mask_path::String)
     lock(REGISTRY_LOCK) do
-        registry = if isfile(registry_path)
+        registry = load_registry(registry_path)
-            try
-                JSON.parsefile(registry_path, dicttype=Dict{String,Any})
-            catch e
-                @error "Failed to parse registry.json while updating mask fields: $e"
-                Dict{String,Any}()
-            end
-        else
-            Dict{String,Any}()
-        end
         
         if haskey(registry, dataset_name)
             registry[dataset_name]["has_mask"] = has_mask
             registry[dataset_name]["mask_path"] = mask_path
         else
-            # Create new entry if it doesn't exist
             registry[dataset_name] = Dict{String,Any}(
                 "has_mask" => has_mask,
                 "mask_path" => mask_path,
@@ -1073,13 +1211,7 @@ function update_registry_mask_fields(registry_path, dataset_name, has_mask, mask
             )
         end
 
-        try
+        save_registry(registry_path, registry)
-            open(registry_path, "w") do f
-                JSON.print(f, registry, 4)
-            end
-        catch e
-            @error "Failed to write to registry.json: $e"
-        end
     end
 end
 

mask.jl

@@ -9,10 +9,12 @@ using Statistics, NaturalSort, LinearAlgebra, StipplePlotly
 using Base.Filesystem: mv
 
 using MSI_src
-using .MSI_src: MSIData, process_image_pipeline
+using .MSI_src: MSIData, process_image_pipeline, REGISTRY_LOCK
 
 # Plot Handling
-include("./julia_imzML_visual.jl")
+if !@isdefined(increment_image)
+    include("./julia_imzML_visual.jl")
+end
 
 # Image Processing Pipeline
 using ImageBinarization
@@ -150,6 +152,7 @@ end
     @in selected_folder_main = ""
     @out available_folders = String[]
     @out image_available_folders = String[]
+    @out registry_path = abspath(joinpath(@__DIR__, "public", "registry.json"))
     @private registry_init_done = false
     @in refetch_folders = false
 
@@ -190,6 +193,7 @@ end
     @in btn_flip_mask = false
     @in btn_save_final_mask = false
     @in btn_move_mask = false
+    @in btn_upload_premade_mask = false
 
     # --- Manual Editor State ---
     @in brush_size = 10
@@ -271,6 +275,33 @@ end
                 msgimg = "No MSI images found in this dataset."
                 show_verification_plot = false
             end
+            
+            # --- Check for mask in registry and update message ---
+            reg_path = abspath(joinpath(@__DIR__, "public", "registry.json"))
+            if isfile(reg_path)
+                registry = JSON.parsefile(reg_path) # Assuming JSON.jl is loaded
+                if haskey(registry, selected_folder_main)
+                    dataset_entry = registry[selected_folder_main]
+                    if get(dataset_entry, "has_mask", false) && haskey(dataset_entry, "mask_path")
+                        mask_filepath = dataset_entry["mask_path"]
+                        mask_filename = basename(mask_filepath)
+                        mask_editor_message = "Dataset has an existing mask: $(mask_filename)"
+                        # Optionally, load this mask if the user is currently editing
+                        if is_editing_mask && isempty(smoothed_mask_path)
+                             # Only auto-load if not already editing a mask and smoothed_mask_path is empty
+                            smoothed_mask_path = get_timestamped_path("/css/masks/$(mask_filename)")
+                            plotdata_verify, plotlayout_verify, show_verification_plot = update_main_plot(imgInt, smoothed_mask_path, imgTrans)
+                            mask_editor_message *= " (loaded for editing)"
+                        end
+                    else
+                        mask_editor_message = "No existing mask found for this dataset."
+                    end
+                else
+                    mask_editor_message = "Dataset entry not found in registry."
+                end
+            else
+                mask_editor_message = "Registry file (registry.json) not found."
+            end
         end
     end
     
@@ -345,6 +376,86 @@ end
         end
     end
 
+    @onbutton btn_upload_premade_mask begin
+        picked_path = pick_file(; filterlist="png,bmp,jpg,jpeg")
+        if !isempty(picked_path)
+            target_dir = joinpath("public", "css", "masks")
+            mkpath(target_dir)
+            
+            uploaded_filename = "premade_mask.png"
+            destination_path = joinpath(target_dir, uploaded_filename)
+            
+            local_uploaded_path = "" # Define here for finally block cleanup
+            try
+                cp(picked_path, destination_path; force=true)
+                local_uploaded_path = destination_path # Full path to the copied image
+                
+                # --- Binary Check ---
+                img = load(local_uploaded_path)
+                gray_img = ensure_grayscale(img)
+                
+                # Check if image is effectively binary (only two distinct values, or close enough)
+                # Convert to raw pixel values for robust checking
+                pixel_values = Float64.(collect(Iterators.flatten(channelview(gray_img))))
+                unique_pixel_values = unique(pixel_values)
+                
+                # Heuristic: if more than 2 distinct values exist, it's not truly binary
+                # Allow for floating point inaccuracies
+                if length(unique_pixel_values) > 2
+                    # Find min and max for range check
+                    min_val, max_val = extrema(unique_pixel_values)
+                    # If values are not clustered at 0 and 1, it's likely not binary
+                    if !(min_val < 0.1 && max_val > 0.9 && all(v -> v < 0.1 || v > 0.9, unique_pixel_values))
+                        mask_editor_message = "Warning: Uploaded image is not purely binary (found $(length(unique_pixel_values)) distinct pixel values). Please use 'Upload to Create Mask' for processing or ensure your image is strictly black and white."
+                        mask_editor_warning = true
+                        rm(local_uploaded_path; force=true) # Clean up non-binary file
+                        return
+                    end
+                end
+
+                # If it has 2 values, or only one value, ensure it's a true BitMatrix
+                # This handles cases like 0-255 images being converted to 0.0-1.0 and then to true/false
+                binary_img = gray_img .>= 0.5 # Ensure pure binary (true/false)
+                save(local_uploaded_path, binary_img)
+                
+                # --- Resizing to Slice (if imgInt is present) ---
+                slice_path_cleaned = replace(imgInt, r"\?.*" => "")
+                slice_full_path = joinpath("public", lstrip(slice_path_cleaned, '/'))
+
+                if !isempty(imgInt) && isfile(slice_full_path) # Only resize if a slice is present
+                    slice_img = load(slice_full_path)
+                    mask_img = load(local_uploaded_path) # Reload potentially re-binarized image to get correct type/size
+                    
+                    if size(slice_img) != size(mask_img)
+                        @info "Resizing pre-made mask to match slice dimensions."
+                        resized_mask = imresize(mask_img, size(slice_img))
+                        save(local_uploaded_path, resized_mask)
+                    end
+                end
+                
+                is_browsing_slices = false
+                is_editing_mask = true
+                # Update smoothed_mask_path to the new file, with timestamp for cache busting
+                relative_path_for_web = "/css/masks/$(uploaded_filename)"
+                smoothed_mask_path = get_timestamped_path(relative_path_for_web)
+                mask_editor_message = "Pre-made mask uploaded and loaded successfully!"
+                mask_editor_warning = false
+                
+                # Trigger plot update
+                plotdata_verify, plotlayout_verify, show_verification_plot = update_main_plot(imgInt, smoothed_mask_path, imgTrans)
+
+            catch e
+                @error "Failed to upload pre-made mask" exception=(e, catch_backtrace())
+                mask_editor_message = "Error uploading pre-made mask: $(sprint(showerror, e))"
+                mask_editor_warning = true
+                if !isempty(local_uploaded_path) && isfile(local_uploaded_path) rm(local_uploaded_path; force=true) end # Clean up
+            end
+        else
+            mask_editor_message = "No file selected for upload."
+            mask_editor_warning = true
+        end
+    end
+
     @onbutton btn_change_slice begin
         is_browsing_slices = true
         is_editing_mask = false
@@ -690,9 +801,7 @@ end
                 end
                 
                 # Save the updated registry
-                open(reg_path, "w") do f
+                save_registry(reg_path, registry)
-                    JSON.print(f, registry, 4)
-                end
                 
                 @info "Registry updated with mask: $(final_mask_name)"
                 
@@ -765,9 +874,7 @@ end
 
                 if !isempty(new_folders) || !isempty(removed_folders)
                     println("Registry changed, saving...")
-                    open(reg_path, "w") do f
+                    save_registry(reg_path, registry)
-                        JSON.print(f, registry, 4)
-                    end
                 end
                 
                 all_folders = sort(collect(keys(registry)), lt=natural)

mask.jl.html

@@ -26,6 +26,8 @@
                     :disable="!is_browsing_slices || !imgInt"></q-btn>
                 <q-btn class="q-ma-sm btn-style" v-on:click="btn_upload_mask = true" label="Upload to Create Mask"
                     :disable="!is_browsing_slices || !imgInt"></q-btn>
+                <q-btn class="q-ma-sm btn-style" v-on:click="btn_upload_premade_mask = true" label="Upload Pre-made Mask"
+                    :disable="!is_browsing_slices"></q-btn>
                 <q-btn class="q-ma-sm btn-style" v-on:click="btn_change_slice = true" label="Change Mask Blueprint"
                     :disable="!is_editing_mask"></q-btn>
             </div>

precompile_script.jl

@@ -0,0 +1,173 @@
+# precompile_script.jl
+# This script exercises the core MSI processing kernels (imzML and mzML) 
+# to ensure they are precompiled into the custom system image.
+
+using MSI_src
+using Mmap
+using Base64
+using PlotlyBase
+
+# --- DYNAMIC WARM-UP WORKLOAD --- #
+
+function create_warmup_datasets(dir, T_mz::Type, T_int::Type)
+    # 1. Create minimal imzML/ibd
+    imzml_path = joinpath(dir, "warmup_$(T_mz)_$(T_int).imzML")
+    ibd_path = joinpath(dir, "warmup_$(T_mz)_$(T_int).ibd")
+    
+    n_points = 10
+    n_pixels = 1
+    mz_bytes = n_points * sizeof(T_mz)
+    int_bytes = n_points * sizeof(T_int)
+    
+    # Write some dummy binary data (m/z must be sorted for validation)
+    open(ibd_path, "w") do f
+        write(f, sort(rand(T_mz, n_points)))
+        write(f, rand(T_int, n_points))
+    end
+    
+    # Minimal but VALID imzML structure that passes axes_config_img
+    imzml_content = """<?xml version="1.0" encoding="utf-8"?>
+<mzML xmlns="http://psi.hupo.org/ms/mzML" version="1.1.0">
+  <cvList count="2">
+    <cv id="MS" fullName="Proteomics Standards Initiative Mass Spectrometry Ontology" version="3.30.0" URI="http://psidev.cvs.sourceforge.net/*checkout*/psidev/ms/mzML/ontology/psi-ms.obo"/>
+    <cv id="IMS" fullName="Imaging MS Ontology" version="0.9.1" URI="http://www.imzml.org/ontology/imzML1.1.0.obo"/>
+  </cvList>
+  <fileDescription>
+    <fileContent>
+      <cvParam cvRef="IMS" accession="IMS:1000031" name="processed" value=""/>
+      <cvParam cvRef="IMS" accession="IMS:1000080" name="universally unique identifier" value="00000000-0000-0000-0000-000000000000"/>
+      <cvParam cvRef="IMS" accession="IMS:1000091" name="ibd MD5 HTTP" value="00000000000000000000000000000000"/>
+    </fileContent>
+  </fileDescription>
+  <referenceableParamGroupList count="2">
+    <referenceableParamGroup id="mz_array_settings">
+      <cvParam cvRef="MS" accession="MS:1000514" name="m/z array" value=""/>
+      <cvParam cvRef="MS" accession="MS:$(T_mz == Float64 ? "1000523" : "1000521")" name="$(T_mz == Float64 ? "64-bit float" : "32-bit float")" value=""/>
+      <cvParam cvRef="MS" accession="MS:1000574" name="zlib compression" value=""/>
+    </referenceableParamGroup>
+    <referenceableParamGroup id="intensity_array_settings">
+      <cvParam cvRef="MS" accession="MS:1000515" name="intensity array" value=""/>
+      <cvParam cvRef="MS" accession="MS:$(T_int == Float64 ? "1000523" : "1000521")" name="$(T_int == Float64 ? "64-bit float" : "32-bit float")" value=""/>
+      <cvParam cvRef="MS" accession="MS:1000574" name="zlib compression" value=""/>
+    </referenceableParamGroup>
+  </referenceableParamGroupList>
+  <run id="run_1">
+    <spectrumList count="1">
+      <spectrum index="0" id="scan=1" defaultArrayLength="$n_points">
+        <cvParam cvRef="IMS" accession="IMS:1000050" name="coordinate x" value="1"/>
+        <cvParam cvRef="IMS" accession="IMS:1000051" name="coordinate y" value="1"/>
+        <binaryDataArrayList count="2">
+          <binaryDataArray encodedLength="0">
+            <referenceableParamGroupRef ref="mz_array_settings"/>
+            <cvParam cvRef="IMS" accession="IMS:1000102" name="external data" value=""/>
+            <cvParam cvRef="IMS" accession="IMS:1000103" name="external offset" value="0"/>
+            <cvParam cvRef="IMS" accession="IMS:1000104" name="external array length" value="$n_points"/>
+            <cvParam cvRef="IMS" accession="IMS:1000101" name="external encoded length" value="$mz_bytes"/>
+            <binary/>
+          </binaryDataArray>
+          <binaryDataArray encodedLength="0">
+            <referenceableParamGroupRef ref="intensity_array_settings"/>
+            <cvParam cvRef="IMS" accession="IMS:1000102" name="external data" value=""/>
+            <cvParam cvRef="IMS" accession="IMS:1000103" name="external offset" value="$mz_bytes"/>
+            <cvParam cvRef="IMS" accession="IMS:1000104" name="external array length" value="$n_points"/>
+            <cvParam cvRef="IMS" accession="IMS:1000101" name="external encoded length" value="$int_bytes"/>
+            <binary/>
+          </binaryDataArray>
+        </binaryDataArrayList>
+      </spectrum>
+    </spectrumList>
+  </run>
+</mzML>
+"""
+    write(imzml_path, imzml_content)
+
+    # 2. Create minimal mzML (Base64)
+    mzml_path = joinpath(dir, "warmup_$(T_mz)_$(T_int).mzML")
+    b64_mz = base64encode(sort(rand(T_mz, n_points)))
+    b64_int = base64encode(rand(T_int, n_points))
+    
+    mzml_content = """<?xml version="1.0" encoding="utf-8"?>
+<mzML xmlns="http://psi.hupo.org/ms/mzML" version="1.1.0">
+  <run id="run_1">
+    <spectrumList count="1">
+      <spectrum index="0" id="scan=1" defaultArrayLength="$n_points">
+        <binaryDataArrayList count="2">
+          <binaryDataArray encodedLength="$(length(b64_mz))">
+            <cvParam cvRef="MS" accession="MS:1000514" name="m/z array" value=""/>
+            <cvParam cvRef="MS" accession="MS:$(T_mz == Float64 ? "1000523" : "1000521")" name="" value=""/>
+            <binary>$b64_mz</binary>
+          </binaryDataArray>
+          <binaryDataArray encodedLength="$(length(b64_int))">
+            <cvParam cvRef="MS" accession="MS:1000515" name="intensity array" value=""/>
+            <cvParam cvRef="MS" accession="MS:$(T_int == Float64 ? "1000523" : "1000521")" name="" value=""/>
+            <binary>$b64_int</binary>
+          </binaryDataArray>
+        </binaryDataArrayList>
+      </spectrum>
+    </spectrumList>
+  </run>
+  <indexList count="1"><index name="spectrum"><offset idRef="scan=1">PLACEHOLDER</offset></index></indexList>
+  <indexListOffset>ID_OFFSET</indexListOffset>
+</mzML>
+"""
+    spec_start = findfirst("<spectrum", mzml_content).start - 1
+    index_start = findfirst("<indexList", mzml_content).start - 1
+    mzml_content = replace(mzml_content, "PLACEHOLDER" => string(spec_start))
+    mzml_content = replace(mzml_content, "ID_OFFSET" => string(index_start))
+    
+    write(mzml_path, mzml_content)
+    
+    return imzml_path, mzml_path
+end
+
+println("Starting robust precompilation warmup (imzML + mzML)...")
+
+try
+    mktempdir() do tmp_dir
+        for (T_mz, T_int) in [(Float32, Float32), (Float64, Float32)]
+            println("Exercising pathways for $T_mz and $T_int...")
+            imzml_path, mzml_path = create_warmup_datasets(tmp_dir, T_mz, T_int)
+            
+            try
+                # Exercise imzML pathway
+                imzml_data = MSI_src.load_imzml_lazy(imzml_path; use_mmap=true)
+                MSI_src.get_multiple_mz_slices(imzml_data, [10.0, 20.0], 0.1)
+                
+                # Exercise Sprint 2 Streaming Pipeline
+                config = MSI_src.PipelineConfig(
+                    steps=[
+                        MSI_src.StreamingStep(:baseline_correction, Dict(:method => :snip, :iterations => 5)),
+                        MSI_src.StreamingStep(:normalization, Dict(:method => :tic)),
+                        MSI_src.StreamingStep(:peak_picking, Dict(:method => :profile, :snr_threshold => 3.0))
+                    ],
+                    num_bins=2000
+                )
+                try
+                    MSI_src.process_dataset!(imzml_data, config)
+                catch
+                    # Ignore matrix mismatch errors from tiny random data 
+                end
+                
+                # Exercise mzML pathway
+                mzml_data = MSI_src.load_mzml_lazy(mzml_path)
+                MSI_src.GetSpectrum(mzml_data, 1)
+            catch e
+                @warn "Warmup failed for $T_mz/$T_int: $e"
+            end
+        end
+
+        # --- COMMON KERNELS ---
+        println("Precompiling image processing kernels...")
+        dummy_slice = rand(Float32, 4, 4)
+        MSI_src.TrIQ(dummy_slice, 256, 1.0)
+        MSI_src.save_bitmap(joinpath(tmp_dir, "d.bmp"), zeros(UInt8, 4, 4), MSI_src.ViridisPalette)
+        
+        # Plotly Warmup
+        p = PlotlyBase.Plot(PlotlyBase.scatter(x=1:2, y=[1,2]))
+    end
+catch e
+    if !(e isa InterruptException)
+        @warn "Global Warmup failed: $e"
+    end
+end
+println("Precompilation workload completed.")

public/css/genieapp.css

@@ -41,6 +41,38 @@
     color: rgb(0, 0, 0) !important;
 }
 
+/* Placeholder color for custom-standout q-inputs */
+.custom-standout .q-field__native::placeholder {
+    color: #CFD8DC !important; /* Lighter grey for placeholder text */
+    opacity: 1 !important; /* Ensure full visibility */
+}
+
+.custom-standout .q-field__control::before {
+    border-color: #7f8389 !important; /* Keep the original border color */
+}
+
+/* Placeholder color for various browsers */
+.custom-standout input::placeholder {
+    color: #CFD8DC !important;
+    opacity: 1 !important;
+}
+.custom-standout input::-webkit-input-placeholder { /* WebKit, Blink, Edge */
+    color: #CFD8DC !important;
+    opacity: 1 !important;
+}
+.custom-standout input::-moz-placeholder { /* Mozilla Firefox 19+ */
+    color: #CFD8DC !important;
+    opacity: 1 !important;
+}
+.custom-standout input:-ms-input-placeholder { /* Internet Explorer 10-11 */
+    color: #CFD8DC !important;
+    opacity: 1 !important;
+}
+.custom-standout input::-ms-input-placeholder { /* Microsoft Edge */
+    color: #CFD8DC !important;
+    opacity: 1 !important;
+}
+
 #tabHeader {
     color: #009f90;
 }
@@ -82,10 +114,6 @@
     font-size: 0.9rem;
 }
 
-#intDivStyle-left .q-tab-panels, #intDivStyle-right .q-tab-panels {
-    /* Removed fixed height */
-}
-
 #intDivStyle-left .q-tab-panel, #intDivStyle-right .q-tab-panel {
     height: auto; /* Let content define height */
     overflow-y: hidden; /* Remove scrollbar */
@@ -118,3 +146,24 @@
 .q-option-group > div {
     margin-bottom: 8px;
 }
+
+/* Loading Overlay Styles */
+.loading-overlay {
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background-color: rgba(0, 0, 0, 0.7);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  z-index: 9999;
+  text-align: center;
+}
+.loading-content .q-spinner {
+  color: white;
+}
+.loading-content .text-h6 {
+  color: white;
+}

src/BloomFilters.jl

@@ -20,21 +20,7 @@ mutable struct BloomFilter{T}
     count::Int
 end
 
-"""
-    StreamingBloomFilter
 
-A memory-efficient Bloom filter that doesn't store all values at once.
-"""
-mutable struct StreamingBloomFilter
-    bits::BitVector
-    size::Int
-    hash_count::Int
-    seed::UInt64
-    count::Int
-    # Streaming state
-    current_chunk::Vector{Int}
-    chunk_size::Int
-end
 
 """
     BloomFilter{T}(expected_elements::Int, false_positive_rate::Float64=0.01; kwargs...) -> Return type
@@ -51,6 +37,9 @@ Creates a Bloom filter optimized for the expected number of elements and desired
 
 - `seed::Union{UInt32,UInt64}`: Keyword description
     (**Default**: `0x12345678`)
+
+# Returns
+- A `BloomFilter{T}`.
 """
 function BloomFilter{T}(expected_elements::Int, false_positive_rate::Float64=0.01; seed::Union{UInt32,UInt64}=0x12345678) where T
     # Convert seed to UInt64 for consistency
@@ -73,6 +62,9 @@ Calculates the optimal number of bits for a Bloom filter
 
 - `n::Int`: Expected number of elements
 - `p::Float64`: Desired false positive rate
+
+# Returns
+- An `Int` to determine the optimal number of bits for a Bloom filter.
 """
 function optimal_bit_size(n::Int, p::Float64)::Int
     if p <= 0.0 || p >= 1.0
@@ -91,7 +83,10 @@ Calculates the optimal number of hash functions for a Bloom filter.
 # Arguments
 
 - `n::Int`: Expected number of elements
-- `p::Float64`: Desired false positive rate
+- `m::Int`: Number of bits in the filter
+
+# Returns
+- An `Int` to determine the optimal number of hash functions for a Bloom filter.
 """
 function optimal_hash_count(n::Int, m::Int)::Int
     if n <= 0 || m <= 0
@@ -106,6 +101,16 @@ end
     hash_functions(item::T, count::Int, size::Int, seed::UInt64) -> Vector{Int}
 
 Generates multiple hash values for an item using double hashing technique.
+
+# Arguments
+
+- `item::T`: Argument description
+- `count::Int`: Argument description
+- `size::Int`: Argument description
+- `seed::UInt64`: Argument description
+
+# Returns
+- A `Vector{Int}` of hash values.
 """
 function hash_functions(item::T, count::Int, size::Int, seed::UInt64) where T
     # Use Julia's built-in hash with different seeds
@@ -129,7 +134,7 @@ end
 
 Adds an element to the Bloom filter.
 
-# Arguments
+# Arguments:
 
 - `bf::BloomFilter{T}`: Argument description
 - `item::T`: Argument description
@@ -150,6 +155,11 @@ end
 
 Checks whether an element is possibly in the Bloom filter.
 Returns `true` if the element might be in the set, `false` if it's definitely not.
+
+# Arguments:
+
+- `item::T`: Argument description
+- `bf::BloomFilter{T}`: Argument description
 """
 function Base.in(item::T, bf::BloomFilter{T})::Bool where T
     hashes = hash_functions(item, bf.hash_count, bf.size, bf.seed)
@@ -166,6 +176,11 @@ end
     contains(bf::BloomFilter{T}, item::T) -> Bool
 
 Alias for `in()` for compatibility with your existing code.
+
+# Arguments:
+
+- `bf::BloomFilter{T}`: Argument description
+- `item::T`: Argument description
 """
 contains(bf::BloomFilter{T}, item::T) where T = item in bf
 
@@ -173,6 +188,11 @@ contains(bf::BloomFilter{T}, item::T) where T = item in bf
     add!(bf::BloomFilter{T}, item::T)
 
 Alias for `push!()` for compatibility with your existing code.
+
+# Arguments:
+
+- `bf::BloomFilter{T}`: Argument description
+- `item::T`: Argument description
 """
 add!(bf::BloomFilter{T}, item::T) where T = push!(bf, item)
 
@@ -180,6 +200,10 @@ add!(bf::BloomFilter{T}, item::T) where T = push!(bf, item)
     false_positive_rate(bf::BloomFilter) -> Float64
 
 Estimates the current false positive rate of the Bloom filter.
+
+# Arguments:
+
+- `bf::BloomFilter`: Argument description
 """
 function false_positive_rate(bf::BloomFilter)::Float64
     if bf.count == 0

src/Common.jl

@@ -1,104 +1,105 @@
-# src/Common.jl - Updated with BloomFilter
 using Base.Threads
+using Mmap
+
+# POSIX madvise constants
+const MADV_NORMAL = 0
+const MADV_RANDOM = 1
+const MADV_SEQUENTIAL = 2
+const MADV_WILLNEED = 3
+const MADV_DONTNEED = 4
+
+"""
+    posix_madvise(buffer::AbstractArray, advice::Integer)
+
+A safe wrapper for the OS `madvise` system call. Signals the kernel about the 
+access pattern for a memory-mapped region. Currently supports Linux/Unix systems.
+"""
+function posix_madvise(buffer::AbstractArray, advice::Integer)
+    @static if Sys.isunix()
+        try
+            ptr = pointer(buffer)
+            len = sizeof(buffer)
+            # ccall(:madvise, return_type, (arg_types...), args...)
+            ret = ccall(:madvise, Int32, (Ptr{Cvoid}, Csize_t, Int32), ptr, len, Int32(advice))
+            return ret == 0
+        catch
+            return false
+        end
+    else
+        return false # Not supported on this OS
+    end
+end
 
 # --- Buffer Pooling ---
-"""
-    BufferPool
 
-A thread-safe pool of `Vector{UInt8}` buffers, categorized by their size.
-This helps reduce memory allocations and garbage collection overhead by reusing buffers.
-
-# Fields
-- `lock`: A `ReentrantLock` to ensure thread-safe access to the pool.
-- `buffers`: A dictionary mapping buffer size (in bytes) to a list of available buffers of that size.
-"""
-mutable struct BufferPool
-    lock::ReentrantLock
-    buffers::Dict{Int, Vector{Vector{UInt8}}}
-
-    function BufferPool()
-        new(ReentrantLock(), Dict{Int, Vector{Vector{UInt8}}}())
-    end
-end
-
-"""
-    get_buffer!(pool::BufferPool, size::Int) -> Vector{UInt8}
-
-Retrieves a `Vector{UInt8}` buffer of at least `size` bytes from the pool.
-If no suitable buffer is available, a new one is allocated.
-
-# Arguments
-- `pool`: The `BufferPool` to retrieve the buffer from.
-- `size`: The minimum desired size of the buffer in bytes.
-
-# Returns
-- A `Vector{UInt8}` buffer.
-"""
-function get_buffer!(pool::BufferPool, size::Int)::Vector{UInt8}
-    lock(pool.lock) do
-        # Find an existing buffer that is large enough
-        for (buffer_size, buffer_list) in pool.buffers
-            if buffer_size >= size && !isempty(buffer_list)
-                buffer = pop!(buffer_list)
-                # Resize if necessary (and if it's significantly larger than needed)
-                if length(buffer) > size * 2 # Heuristic: if buffer is more than twice the requested size
-                    return Vector{UInt8}(undef, size) # Allocate new smaller buffer
-                else
-                    return buffer
-                end
-            end
-        end
-        # No suitable buffer found, allocate a new one
-        return Vector{UInt8}(undef, size)
-    end
-end
-
-"""
-    release_buffer!(pool::BufferPool, buffer::Vector{UInt8})
-
-Returns a `Vector{UInt8}` buffer to the pool for future reuse.
-
-# Arguments
-- `pool`: The `BufferPool` to return the buffer to.
-- `buffer`: The `Vector{UInt8}` buffer to release.
-"""
-function release_buffer!(pool::BufferPool, buffer::Vector{UInt8})
-    lock(pool.lock) do
-        buffer_size = length(buffer)
-        if !haskey(pool.buffers, buffer_size)
-            pool.buffers[buffer_size] = Vector{Vector{UInt8}}()
-        end
-        push!(pool.buffers[buffer_size], buffer)
-    end
-    return nothing
-end
 
 
 """
     SimpleBufferPool
 
 A dramatically simplified buffer pool that avoids complex locking.
+
+# Arguments:
+- `buffers::Dict{Int, Vector{Vector{UInt8}}}`: A dictionary mapping buffer size (in bytes) to a list of available buffers of that size.
+- `max_pool_size::Int`: The maximum number of buffers to keep in the pool.
+- `lock::ReentrantLock`: A lock to ensure thread-safe access to the pool.
 """
 mutable struct SimpleBufferPool
     buffers::Dict{Int, Vector{Vector{UInt8}}}
     max_pool_size::Int
+    lock::ReentrantLock  # Add lock for thread safety
 end
 
-SimpleBufferPool() = SimpleBufferPool(Dict{Int, Vector{Vector{UInt8}}}(), 50)
+"""
+    SimpleBufferPool()
 
+Creates a new `SimpleBufferPool` with a default maximum pool size of 50.
+"""
+SimpleBufferPool() = SimpleBufferPool(Dict{Int, Vector{Vector{UInt8}}}(), 50, ReentrantLock())
+
+"""
+    get_buffer!(pool::SimpleBufferPool, size::Int) -> Vector{UInt8}
+
+Retrieves a `Vector{UInt8}` buffer of at least `size` bytes from the pool.
+If no suitable buffer is available, a new one is allocated.
+
+# Arguments:
+- `pool`: The `SimpleBufferPool` to retrieve the buffer from.
+- `size`: The minimum desired size of the buffer in bytes.
+
+# Returns:
+- A `Vector{UInt8}` buffer.
+"""
 function get_buffer!(pool::SimpleBufferPool, size::Int)::Vector{UInt8}
+    buf = lock(pool.lock) do
         # Check for existing buffers of exact size first
         if haskey(pool.buffers, size) && !isempty(pool.buffers[size])
             return pop!(pool.buffers[size])
         end
+        return nothing
+    end
     
-    # No suitable buffer found, allocate new one
+    if buf !== nothing
+        return buf
+    end
+    
+    # No suitable buffer found, allocate new one (outside lock to reduce contention)
     return Vector{UInt8}(undef, size)
 end
 
+"""
+    release_buffer!(pool::SimpleBufferPool, buffer::Vector{UInt8})
+
+Returns a `Vector{UInt8}` buffer to the pool for future reuse.
+
+# Arguments:
+- `pool`: The `SimpleBufferPool` to return the buffer to.
+- `buffer`: The `Vector{UInt8}` buffer to release.
+"""
 function release_buffer!(pool::SimpleBufferPool, buffer::Vector{UInt8})
     size = length(buffer)
     
+    lock(pool.lock) do
         if !haskey(pool.buffers, size)
             pool.buffers[size] = Vector{Vector{UInt8}}()
         end
@@ -107,6 +108,7 @@ function release_buffer!(pool::SimpleBufferPool, buffer::Vector{UInt8})
         if length(pool.buffers[size]) < pool.max_pool_size
             push!(pool.buffers[size], buffer)
         end
+    end
     # If pool is full, let buffer get GC'd
 end
 
@@ -127,15 +129,26 @@ struct InvalidSpectrumError <: MSIError
 end
 
 # --- Shared Constants ---
-const DEFAULT_CACHE_SIZE = 100
+const DEFAULT_CACHE_SIZE = 100 # Default cache size for spectra
-const DEFAULT_NUM_BINS = 2000
+const DEFAULT_NUM_BINS = 2000 # Default number of bins for spectra
-const DEFAULT_BLOOM_FILTER_SIZE = 1000  # Default size for empty spectra
+const DEFAULT_BLOOM_FILTER_SIZE = 10000  # Default size for empty spectra
-const DEFAULT_FALSE_POSITIVE_RATE = 0.01
+const DEFAULT_FALSE_POSITIVE_RATE = 0.01 # Default false positive rate for bloom filters
 
 """
     validate_spectrum_data(mz, intensity, id)
 
 Checks a spectrum for common data integrity issues.
+
+# Arguments:
+- `mz`: The m/z values of the spectrum.
+- `intensity`: The intensity values of the spectrum.
+- `id`: The ID of the spectrum.
+
+# Returns:
+- `true` if the spectrum is valid.
+
+# Throws:
+- `InvalidSpectrumError` if the spectrum is invalid.
 """
 function validate_spectrum_data(mz::AbstractVector, intensity::AbstractVector, id)
     if length(mz) != length(intensity)
@@ -158,6 +171,12 @@ end
 
 Memory-optimized version that processes data in chunks to reduce temporary allocations
 and avoid holding large intermediate arrays.
+
+# Arguments:
+- `mz`: The m/z values of the spectrum.
+
+# Returns:
+- A `BloomFilter{Int}` for the spectrum.
 """
 function create_bloom_filter_for_spectrum(mz::AbstractVector{<:Real})::BloomFilter{Int}
     if isempty(mz)
@@ -188,6 +207,9 @@ end
     empty_bloom_filter() -> BloomFilter{Float64}
 
 Creates an empty Bloom filter for spectra with no data.
+
+# Returns:
+- An empty `BloomFilter{Float64}`.
 """
 function empty_bloom_filter()::BloomFilter{Int}
     return BloomFilter{Int}(size=DEFAULT_BLOOM_FILTER_SIZE, hash_count=3)
@@ -197,6 +219,12 @@ end
     AtomicFlag
 
 A thread-safe boolean flag using atomic operations.
+
+# Fields:
+- `value::Base.Threads.Atomic{Int}`: The atomic value of the flag.
+
+# Arguments:
+- `value::Base.Threads.Atomic{Int}`: The atomic value of the flag.
 """
 struct AtomicFlag
     value::Base.Threads.Atomic{Int}

src/MSI_src.jl

@@ -19,7 +19,22 @@ export OpenMSIData,
        get_global_mz_range,
        MSIData,
        _iterate_spectra_fast,
-       validate_spectrum
+       validate_spectrum,
+       get_mz_slice,
+       REGISTRY_LOCK,
+       SpectrumMetadata,
+       SpectrumAsset,
+       SpectrumMode,
+       CENTROID,
+       PROFILE,
+       MzMLSource,
+       ImzMLSource,
+       SimpleBufferPool,
+       get_buffer!,
+       release_buffer!
+
+# Define shared registry lock
+const REGISTRY_LOCK = ReentrantLock()
 
 # Export the public preprocessing & precalculations API
 export run_preprocessing_analysis,
@@ -30,23 +45,47 @@ export run_preprocessing_analysis,
        BaselineCorrection,
        Normalization,
        PeakPicking,
-       PeakBinningParams,
+       PeakBinning,
        get_masked_spectrum_indices,
-       detect_peaks_profile,
+       detect_peaks_profile_core, 
-       detect_peaks_centroid,
+       detect_peaks_centroid_core, 
-       smooth_spectrum,
+       smooth_spectrum_core, 
-       apply_baseline_correction,
+       apply_baseline_correction_core, 
-       apply_normalization,
+       apply_normalization_core, 
-       bin_peaks,
+       bin_peaks_core, 
        PeakSelection,
        PeakAlignment,
-       find_calibration_peaks,
+       find_calibration_peaks_core, 
-       align_peaks_lowess,
+       align_peaks_lowess_core,
-       MutableSpectrum
+       MutableSpectrum,
+       transform_intensity_core,
+       calibrate_spectra_core
+
+export apply_baseline_correction,
+       apply_smoothing,
+       apply_peak_picking,
+       apply_calibration,
+       apply_peak_alignment,
+       apply_normalization,
+       apply_peak_binning,
+       apply_intensity_transformation,
+       save_feature_matrix
+
+# Sprint 2: Streaming Pipeline API
+export process_dataset!,
+       PipelineConfig,
+       StreamingStep,
+       normalize_inplace!,
+       transform_inplace!,
+       smooth_inplace!,
+       baseline_subtract_inplace!,
+       detect_peaks_streaming,
+       calibrate_inplace!
 
 # Include all source files directly into the main module
 include("BloomFilters.jl")
 include("Common.jl")
+include("ResourcePool.jl")
 include("MSIData.jl")
 include("ParserHelpers.jl")
 include("mzML.jl")
@@ -55,6 +94,9 @@ include("MzmlConverter.jl")
 include("Preprocessing.jl")
 include("ImageProcessing.jl")
 include("Precalculations.jl")
+include("PreprocessingPipeline.jl")
+include("StreamingKernels.jl")
+include("StreamingPipeline.jl")
 
 using Setfield # For immutable struct updates
 
@@ -67,15 +109,28 @@ using Setfield # For immutable struct updates
 Opens a .mzML or .imzML file and prepares it for data access.
 
 This is the main entry point for the new data access API.
+
+# Arguments
+
+- `filepath::String`: Path to the .mzML or .imzML file to open.
+- `cache_size::Int`: Number of spectra to cache in memory.
+- `spectrum_type_map::Union{Dict{Int, Symbol}, Nothing}`: Optional mapping of spectrum indices to types.
+
+# Returns
+- An `MSIData` object.
 """
 function OpenMSIData(filepath::String; cache_size=300, spectrum_type_map::Union{Dict{Int, Symbol}, Nothing}=nothing)
+    # Apply standard path normalization for cross-platform compatibility
+    # Ensure Windows backslashes are converted to OS-native separators
+    norm_filepath = normpath(replace(filepath, "\\" => "/"))
+    
     local msi_data
-    if endswith(lowercase(filepath), ".mzml")
+    if endswith(lowercase(norm_filepath), ".mzml")
-        msi_data = load_mzml_lazy(filepath, cache_size=cache_size)
+        msi_data = load_mzml_lazy(norm_filepath, cache_size=cache_size)
-    elseif endswith(lowercase(filepath), ".imzml")
+    elseif endswith(lowercase(norm_filepath), ".imzml")
-        msi_data = load_imzml_lazy(filepath, cache_size=cache_size)
+        msi_data = load_imzml_lazy(norm_filepath, cache_size=cache_size)
     else
-        error("Unsupported file type: $filepath. Please provide a .mzML or .imzML file.")
+        error("Unsupported file type: $norm_filepath. Please provide a .mzML or .imzML file.")
     end
 
     # Apply spectrum type map if provided

src/MzmlConverter.jl

@@ -842,6 +842,16 @@ Main workflow function to convert a .mzML file to an .imzML file.
 * `target_file`: Path for the output .imzML file (the .ibd will be named accordingly).
 * `img_width`: width dimention for the creation of the x axis
 * `img_height`: height dimention for the creation of the y axis
+
+# Returns
+- `true` if the conversion was successful.
+- `false` if the conversion failed.
+
+# Example
+
+```julia
+ImportMzmlFile("test.mzML", "test.txt", "test.imzML")
+```
 """
 function ImportMzmlFile(source_file::String, sync_file::String, target_file::String; img_width::Int=0, img_height::Int=0)
     if !isfile(source_file)

src/ParserHelpers.jl

@@ -28,8 +28,18 @@ end
 
 Reads a stream line-by-line until a line matches the provided regex.
 
+# Arguments
+- `stream::IO`: The stream to parse the metadata from.
+- `regex::Regex`: The regex to match.
+
 # Returns
 - A `RegexMatch` object if a match is found, otherwise throws an error.
+
+# Example
+
+```julia
+find_tag(open("test.mzML"), r"<spectrum")
+```
 """
 function find_tag(stream, regex::Regex)
     while !eof(stream)
@@ -47,8 +57,18 @@ end
 
 Retrieves an attribute's value from an XML tag string.
 
+# Arguments
+- `source::String`: The string to parse the attribute from.
+- `tag::String`: The tag to match.
+
 # Returns
 - A `RegexMatch` object containing the attribute and its value.
+
+# Example
+
+```julia
+get_attribute("<spectrum index=\"1\">", "index")
+```
 """
 function get_attribute(source::AbstractString, tag::String = "([^=]+)")
     # Construct the regex pattern string
@@ -65,10 +85,16 @@ struct. It reads accessions to determine the data format (e.g., `Float32`),
 compression status (`zlib`), and axis type (m/z vs. intensity).
 
 # Arguments
-- `stream`: An IO stream positioned at the start of the `cvParam` block.
+- `stream::IO`: An IO stream positioned at the start of the `cvParam` block.
 
 # Returns
 - A `SpecDim` struct populated with the parsed configuration.
+
+# Example
+
+```julia
+configure_spec_dim(open("test.mzML"))
+```
 """
 function configure_spec_dim(stream)
     axis = SpecDim(Float64, false, 1, 0, UNKNOWN)  # Add UNKNOWN as default mode

src/Precalculations.jl

@@ -1,14 +1,26 @@
 using StatsBase # For mean, std, median, quantile, mad
+using Base.Threads # For atomic counters and locking
 
 # =============================================================================
 # 7) Spatial & Advanced Processing (Stubs & New Functions)
 # =============================================================================
 
 """
-    find_ppm_error_by_region(msi_data, region_masks, reference_peaks)::Dict
+    find_ppm_error_by_region(msi_data::MSIData, region_masks::Dict, reference_peaks::Dict) -> Dict
 
-Calculates PPM error statistics for different spatial regions.
+Calculates and reports mass accuracy (PPM error) statistics for different spatial regions
-`region_masks` is a Dict mapping region names (e.g., :tumor) to BitMatrix masks.
+defined by masks. This is useful for identifying spatial variations in calibration.
+
+# Arguments
+- `msi_data::MSIData`: The main MSI data object.
+- `region_masks::Dict{Symbol, BitMatrix}`: A dictionary mapping region names (e.g., `:tumor`, `:stroma`)
+  to `BitMatrix` masks. The dimensions of each mask must match `msi_data.image_dims`.
+- `reference_peaks::Dict{Float64, String}`: A dictionary of known reference peaks, mapping
+  theoretical m/z to a name.
+
+# Returns
+- `Dict{Symbol, NamedTuple}`: A dictionary where keys are region names and values are `NamedTuple`s
+  containing the mass accuracy report for that region, as generated by `analyze_mass_accuracy`.
 """
 function find_ppm_error_by_region(msi_data::MSIData, region_masks::Dict, reference_peaks::Dict)
     regional_reports = Dict{Symbol, NamedTuple}()
@@ -37,6 +49,28 @@ function find_ppm_error_by_region(msi_data::MSIData, region_masks::Dict, referen
     return regional_reports
 end
 
+"""
+    analyze_mass_accuracy(msi_data, reference_peaks; ...) -> NamedTuple
+
+Analyzes the mass accuracy for a given subset of spectra by comparing detected peaks
+against a list of known reference masses.
+
+# Arguments
+- `msi_data::MSIData`: The main MSI data object.
+- `reference_peaks::Dict{Float64, String}`: A dictionary of known reference peaks, mapping
+  theoretical m/z to a name.
+- `spectrum_indices::AbstractVector{Int}`: A vector of indices for the spectra to be analyzed.
+- `peak_detection_snr_threshold::Float64`: The Signal-to-Noise ratio threshold to use for
+  detecting peaks within the spectra.
+- `ppm_tolerance_for_matching::Float64`: The tolerance in Parts Per Million (PPM) used to match
+  a detected peak to a reference peak.
+
+# Returns
+- `NamedTuple`: A report containing summary statistics of the PPM errors found, including:
+  - `mean_ppm_error`, `median_ppm_error`, `std_ppm_error`, `min_ppm_error`, `max_ppm_error`
+  - `total_matched_peaks`: The total count of successful matches between detected and reference peaks.
+  - `total_spectra_analyzed`: The number of spectra processed.
+"""
 function analyze_mass_accuracy(
     msi_data::MSIData,
     reference_peaks::Dict{Float64, String}; # m/z => name
@@ -47,17 +81,18 @@ function analyze_mass_accuracy(
     println("Analyzing mass accuracy for $(length(spectrum_indices)) spectra...")
 
     all_ppm_errors = Float64[]
-    total_matched_peaks = 0
+    total_matched_peaks = Base.Threads.Atomic{Int}(0)
-    total_spectra_processed = 0
+    total_spectra_processed = Base.Threads.Atomic{Int}(0)
+    results_lock = ReentrantLock()
 
     _iterate_spectra_fast(msi_data, spectrum_indices) do idx, mz, intensity
-        total_spectra_processed += 1
+        Base.Threads.atomic_add!(total_spectra_processed, 1)
         if !validate_spectrum(mz, intensity)
             @warn "Spectrum $idx is invalid, skipping mass accuracy analysis for it."
             return
         end
 
-        detected_peaks = detect_peaks_profile(mz, intensity; snr_threshold=peak_detection_snr_threshold)
+        detected_peaks = detect_peaks_profile_core(mz, intensity; snr_threshold=peak_detection_snr_threshold)
 
         for ref_mz in keys(reference_peaks)
             # Find the closest detected peak to this reference m/z within tolerance
@@ -73,8 +108,10 @@ function analyze_mass_accuracy(
             end
 
             if best_matched_peak_mz !== nothing
+                lock(results_lock) do
                     push!(all_ppm_errors, min_ppm_error)
-                total_matched_peaks += 1
+                end
+                Base.Threads.atomic_add!(total_matched_peaks, 1)
             end
         end
     end
@@ -88,7 +125,8 @@ function analyze_mass_accuracy(
             min_ppm_error = NaN,
             max_ppm_error = NaN,
             total_matched_peaks = 0,
-            total_spectra_analyzed = total_spectra_processed
+            total_spectra_analyzed = total_spectra_processed,
+            ppm_error_distribution = Float64[]
         )
     end
 
@@ -106,8 +144,9 @@ function analyze_mass_accuracy(
         std_ppm_error = std_err,
         min_ppm_error = min_err,
         max_ppm_error = max_err,
-        total_matched_peaks = total_matched_peaks,
+        total_matched_peaks = total_matched_peaks[],
-        total_spectra_analyzed = total_spectra_processed
+        total_spectra_analyzed = total_spectra_processed[],
+        ppm_error_distribution = all_ppm_errors
     )
 end
 
@@ -118,7 +157,16 @@ end
 """
     calculate_adaptive_bin_tolerance(ppm_error_distribution) -> Float64
 
-Calculates an appropriate binning tolerance based on observed mass accuracy.
+Calculates an appropriate binning tolerance in PPM based on the observed mass accuracy
+distribution. The strategy is to set the tolerance to capture the vast majority of
+peaks from the same analyte, typically using `mean + 3 * standard_deviation`.
+
+# Arguments
+- `ppm_error_distribution::Vector{Float64}`: A vector of PPM error values from a mass
+  accuracy analysis.
+
+# Returns
+- `Float64`: The suggested binning tolerance in PPM, capped between 10.0 and 100.0.
 """
 function calculate_adaptive_bin_tolerance(ppm_error_distribution::Vector{Float64})
     if isempty(ppm_error_distribution)
@@ -148,10 +196,20 @@ function calculate_adaptive_bin_tolerance(ppm_error_distribution::Vector{Float64
 end
 
 """
-    calculate_preprocessing_hints(data::MSIData; sample_size::Int=100)::Dict{Symbol, Any}
+    calculate_preprocessing_hints(data::MSIData; sample_indices)::Dict{Symbol, Any}
 
-Analyzes a sample of spectra to determine optimal default parameters for
+Analyzes a sample of spectra to determine initial "hints" for preprocessing parameters.
-preprocessing steps, returning a dictionary of hints.
+This function provides quick, data-driven defaults for noise level, SNR, and smoothing.
+
+# Arguments
+- `data::MSIData`: The main MSI data object.
+- `sample_indices::AbstractVector{Int}`: The indices of spectra to sample for the analysis.
+
+# Returns
+- `Dict{Symbol, Any}`: A dictionary of hints, including:
+  - `:estimated_noise`: The mean noise level estimated using Median Absolute Deviation (MAD).
+  - `:suggested_snr`: A default SNR threshold (typically 3.0).
+  - `:suggested_smoothing_window`: A suggested window size for smoothing, based on instrument resolution if available.
 """
 function calculate_preprocessing_hints(data::MSIData; sample_indices::AbstractVector{Int})::Dict{Symbol, Any}
     println("Calculating preprocessing hints from a sample of $(length(sample_indices)) spectra...")
@@ -218,9 +276,22 @@ function calculate_preprocessing_hints(data::MSIData; sample_indices::AbstractVe
 end
 
 """
-    analyze_instrument_characteristics(msi_data::MSIData)::Dict
+    analyze_instrument_characteristics(msi_data::MSIData; sample_indices)::Dict
 
-Analyzes instrument metadata and data characteristics to determine acquisition properties.
+Analyzes instrument metadata and spectral data to infer key acquisition properties.
+It combines information from the `msi_data.instrument_metadata` with direct analysis
+of the spectra.
+
+# Arguments
+- `msi_data::MSIData`: The main MSI data object.
+- `sample_indices::AbstractVector{Int}`: The indices of spectra to sample for the analysis.
+
+# Returns
+- `Dict{Symbol, Any}`: A dictionary summarizing instrument characteristics:
+  - `:acquisition_mode`: Inferred as `:profile`, `:centroid`, or `:mixed`.
+  - `:mz_axis_type`: Inferred as `:regular` or `:irregular` based on m/z step consistency.
+  - `:dynamic_range`: An estimate of the intensity dynamic range in orders of magnitude.
+  - Other fields from `instrument_metadata` like `:resolution`, `:polarity`, etc.
 """
 function analyze_instrument_characteristics(msi_data::MSIData; sample_indices::AbstractVector{Int})::Dict
     results = Dict{Symbol, Any}()
@@ -251,23 +322,31 @@ function analyze_instrument_characteristics(msi_data::MSIData; sample_indices::A
         return results
     end
     
+    results_lock = ReentrantLock()
+    
     _iterate_spectra_fast(msi_data, sample_indices) do idx, mz, intensity
         # Record spectrum mode
+        lock(results_lock) do
             push!(spectrum_modes, msi_data.spectra_metadata[idx].mode)
+        end
         
         # Calculate m/z step statistics (for profile data)
         if length(mz) > 1 && msi_data.spectra_metadata[idx].mode == PROFILE
             steps = diff(mz)
             if !isempty(steps)
+                lock(results_lock) do
                     push!(mz_step_sizes, mean(steps))
                 end
             end
+        end
         
         # Record intensity range
         if !isempty(intensity)
+            lock(results_lock) do
                 push!(intensity_ranges, (minimum(intensity), maximum(intensity)))
             end
         end
+    end
     
     # Determine acquisition mode
     if length(spectrum_modes) == 1
@@ -337,9 +416,20 @@ function analyze_instrument_characteristics(msi_data::MSIData; sample_indices::A
 end
 
 """
-    analyze_signal_quality(msi_data::MSIData; sample_size::Int=100)::Dict
+    analyze_signal_quality(msi_data::MSIData; sample_indices)::Dict
 
-Analyzes noise characteristics, signal-to-noise ratios, and overall signal quality.
+Analyzes a sample of spectra to assess signal quality, including noise levels,
+Signal-to-Noise Ratio (SNR), and Total Ion Current (TIC) variation.
+
+# Arguments
+- `msi_data::MSIData`: The main MSI data object.
+- `sample_indices::AbstractVector{Int}`: The indices of spectra to sample for the analysis.
+
+# Returns
+- `Dict{Symbol, Any}`: A dictionary of signal quality metrics:
+  - `:noise_mean`, `:noise_std`, `:noise_cv`: Statistics of the noise level.
+  - `:snr_mean`, `:snr_median`, `:snr_95th`: Distribution statistics of the estimated SNR.
+  - `:tic_mean`, `:tic_std`, `:tic_cv`: Statistics of the Total Ion Current.
 """
 function analyze_signal_quality(msi_data::MSIData; sample_indices::AbstractVector{Int})::Dict
     results = Dict{Symbol, Any}()
@@ -364,11 +454,15 @@ function analyze_signal_quality(msi_data::MSIData; sample_indices::AbstractVecto
         return results
     end
     
+    results_lock = ReentrantLock()
+
     _iterate_spectra_fast(msi_data, sample_indices) do idx, mz, intensity
         if !isempty(intensity)
             # Noise estimation using MAD
             noise = mad(intensity, normalize=true)
+            lock(results_lock) do
                 push!(noise_levels, noise)
+            end
             
             # FIX: More robust SNR calculation
             valid_intensity = intensity[intensity .> 0]  # Remove zeros
@@ -380,14 +474,18 @@ function analyze_signal_quality(msi_data::MSIData; sample_indices::AbstractVecto
                 if noise_robust > 0 && isfinite(signal_estimate)
                     snr_val = signal_estimate / noise_robust
                     # Cap unrealistic SNR values
+                    lock(results_lock) do
                         push!(snr_distribution, min(snr_val, 1e6))
                     end
                 end
+            end
             
             # Total ion count
+            lock(results_lock) do
                 push!(tic_values, sum(intensity))
             end
         end
+    end
     
     if !isempty(noise_levels)
         results[:noise_mean] = mean(noise_levels)
@@ -415,9 +513,21 @@ function analyze_signal_quality(msi_data::MSIData; sample_indices::AbstractVecto
 end
 
 """
-    analyze_mass_accuracy_global(msi_data::MSIData, reference_peaks::Dict, sample_size::Int)::Dict
+    analyze_mass_accuracy_global(msi_data, reference_peaks; spectrum_indices)::Dict
 
-Analyzes mass accuracy across the dataset using reference peaks.
+Performs a global mass accuracy analysis across a sample of spectra and suggests an
+adaptive binning tolerance.
+
+# Arguments
+- `msi_data::MSIData`: The main MSI data object.
+- `reference_peaks::Dict`: A dictionary of known reference peaks.
+- `spectrum_indices::AbstractVector{Int}`: The indices of spectra to sample for the analysis.
+
+# Returns
+- `Dict{Symbol, Any}`: A dictionary containing:
+  - `:global_accuracy`: The `NamedTuple` report from `analyze_mass_accuracy`.
+  - `:suggested_bin_tolerance`: An adaptive tolerance in PPM for peak binning, derived
+    from the mass accuracy results.
 """
 function analyze_mass_accuracy_global(msi_data::MSIData, reference_peaks::Dict; 
                                        spectrum_indices::AbstractVector{Int})::Dict
@@ -431,7 +541,7 @@ function analyze_mass_accuracy_global(msi_data::MSIData, reference_peaks::Dict;
         empty_report = (
             mean_ppm_error = NaN, median_ppm_error = NaN, std_ppm_error = NaN,
             min_ppm_error = NaN, max_ppm_error = NaN, total_matched_peaks = 0,
-            total_spectra_analyzed = 0
+            total_spectra_analyzed = 0, ppm_error_distribution = Float64[]
         )
         results[:global_accuracy] = empty_report
         results[:suggested_bin_tolerance] = 20.0 # Default
@@ -444,7 +554,7 @@ function analyze_mass_accuracy_global(msi_data::MSIData, reference_peaks::Dict;
     
     results[:global_accuracy] = accuracy_report
     results[:suggested_bin_tolerance] = calculate_adaptive_bin_tolerance(
-        collect(Iterators.flatten([accuracy_report.mean_ppm_error]))  # Simplified - would need actual distribution
+        accuracy_report.ppm_error_distribution
     )
     
     println("  - Mean PPM error: $(round(accuracy_report.mean_ppm_error, digits=2))")
@@ -454,9 +564,21 @@ function analyze_mass_accuracy_global(msi_data::MSIData, reference_peaks::Dict;
 end
 
 """
-    analyze_spatial_regions(msi_data::MSIData, region_masks::Dict, reference_peaks::Dict)::Dict
+    analyze_spatial_regions(msi_data, region_masks, reference_peaks)::Dict
 
-Analyzes different spatial regions for variations in signal quality and mass accuracy.
+Analyzes different spatial regions for variations in mass accuracy. This function is a
+wrapper around `find_ppm_error_by_region` and summarizes the results.
+
+# Arguments
+- `msi_data::MSIData`: The main MSI data object.
+- `region_masks::Dict`: A dictionary of named `BitMatrix` masks for each region.
+- `reference_peaks::Dict`: A dictionary of known reference peaks.
+
+# Returns
+- `Dict{Symbol, Any}`: A dictionary containing:
+  - `:regional_ppm_errors`: A dictionary mapping each region name to its mass accuracy report.
+  - `:max_regional_ppm_difference`: The difference between the highest and lowest mean PPM
+    error across all analyzed regions.
 """
 function analyze_spatial_regions(msi_data::MSIData, region_masks::Dict, reference_peaks::Dict)::Dict
     results = Dict{Symbol, Any}()
@@ -484,9 +606,24 @@ function analyze_spatial_regions(msi_data::MSIData, region_masks::Dict, referenc
 end
 
 """
-    analyze_peak_characteristics(msi_data::MSIData, sample_size::Int)::Dict
+    analyze_peak_characteristics(msi_data, instrument_analysis, mass_accuracy_results; spectrum_indices)::Dict
 
-Analyzes peak shape, width, and quality characteristics.
+Analyzes peak shape, width, and quality from a sample of spectra. The behavior
+adapts based on whether the data is in `:profile` or `:centroid` mode.
+
+# Arguments
+- `msi_data::MSIData`: The main MSI data object.
+- `instrument_analysis::Dict`: The output from `analyze_instrument_characteristics`.
+- `mass_accuracy_results`: The output from `analyze_mass_accuracy_global`.
+- `spectrum_indices::AbstractVector{Int}`: The indices of spectra to sample.
+
+# Returns
+- `Dict{Symbol, Any}`: A dictionary of peak metrics:
+  - `:mean_fwhm_ppm`, `:median_fwhm_ppm`: Statistics of Full Width at Half Maximum (FWHM).
+    For centroid data, this is estimated from mass accuracy.
+  - `:mean_gaussian_r2`: The average goodness-of-fit to a Gaussian shape (profile data only).
+  - `:peak_resolution_estimate`: An estimate of instrument resolution based on FWHM.
+  - `:mean_peaks_per_spectrum`: The average number of peaks detected per spectrum.
 """
 function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Dict, mass_accuracy_results; 
                                     spectrum_indices::AbstractVector{Int})::Dict
@@ -508,12 +645,21 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
     if isempty(spectrum_indices)
         @warn "No indices to sample for peak characteristics analysis."
         # Provide sensible defaults if no analysis can be run
+        if acquisition_mode == :centroid
+            # Much more permissive defaults for centroid data
+            results[:mean_fwhm_ppm] = 50.0
+            results[:median_fwhm_ppm] = 50.0
+            results[:mean_gaussian_r2] = 0.0  # Disable shape filtering for centroids
+            results[:peak_resolution_estimate] = 20000.0
+            results[:mean_peaks_per_spectrum] = 1000
+        else
             estimated_fwhm = estimated_mean_ppm_error * (acquisition_mode == :profile ? 3 : 2)
             results[:mean_fwhm_ppm] = estimated_fwhm
             results[:median_fwhm_ppm] = estimated_fwhm
             results[:mean_gaussian_r2] = acquisition_mode == :profile ? 0.7 : 0.9
             results[:peak_resolution_estimate] = 1e6 / estimated_fwhm
             results[:mean_peaks_per_spectrum] = 0
+        end
         return results
     end
 
@@ -525,20 +671,23 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
         peak_counts = Int[]
         fwhm_values = Float64[]
         
-        spectra_analyzed = 0
+        spectra_analyzed = Base.Threads.Atomic{Int}(0)
-        peaks_analyzed = 0
+        peaks_analyzed = Base.Threads.Atomic{Int}(0)
+        results_lock = ReentrantLock()
         
         _iterate_spectra_fast(msi_data, spectrum_indices) do idx, mz, intensity
             if length(mz) < 10  # Skip spectra with too few points
                 return
             end
             
-            spectra_analyzed += 1
+            Base.Threads.atomic_add!(spectra_analyzed, 1)
             meta = msi_data.spectra_metadata[idx]
             
             # Detect peaks with lower SNR threshold to find more peaks
-            peaks = detect_peaks_profile(mz, intensity; snr_threshold=2.0)
+            peaks = detect_peaks_profile_core(mz, intensity; snr_threshold=2.0)
+            lock(results_lock) do
                 push!(peak_counts, length(peaks))
+            end
             
             if !isempty(peaks)
                 # Analyze the strongest 3 peaks per spectrum
@@ -553,15 +702,17 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
                         if !isnan(fwhm_delta_m) && fwhm_delta_m > 0.001 && fwhm_delta_m < 0.5  # Reasonable range in Da
                             fwhm_ppm = 1e6 * fwhm_delta_m / peak.mz
                             if 5.0 < fwhm_ppm < 500.0  # Reasonable ppm range
+                                lock(results_lock) do
                                     push!(peak_widths_ppm, fwhm_ppm)
                                     push!(fwhm_values, fwhm_delta_m)
                                     
                                     r2 = _fit_gaussian_and_r2(mz, intensity, peak_idx, 5)
                                     push!(r_squared_values, r2)
-                                peaks_analyzed += 1
+                                end
+                                Base.Threads.atomic_add!(peaks_analyzed, 1)
                                 
-                                if peaks_analyzed <= 3
+                                if peaks_analyzed[] <= 3
-                                    println("DEBUG: Peak at m/z $(peak.mz), FWHM = $(fwhm_ppm) ppm, R² = $r2")
+                                    println("DEBUG: Peak at m/z $(peak.mz), FWHM = $(fwhm_ppm) ppm, R² = $(r_squared_values[end])")
                                 end
                             end
                         end
@@ -572,7 +723,7 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
             end
         end
         
-        println("DEBUG: Analyzed $peaks_analyzed peaks from $spectra_analyzed spectra")
+        println("DEBUG: Analyzed $(peaks_analyzed[]) peaks from $(spectra_analyzed[]) spectra")
         
         if !isempty(peak_widths_ppm)
             results[:mean_fwhm_ppm] = mean(peak_widths_ppm)
@@ -600,12 +751,14 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
         println("  Analyzing peak characteristics for CENTROID mode from $(length(spectrum_indices)) sample spectra...")
         
         peak_counts = Int[]
-        # peak_intensities = Float64[] # Not strictly needed for these metrics
+        
+        results_lock = ReentrantLock()
 
         _iterate_spectra_fast(msi_data, spectrum_indices) do idx, mz, intensity
             if !isempty(mz)
+                lock(results_lock) do
                     push!(peak_counts, length(mz))
-                # append!(peak_intensities, intensity) # If needed for other metrics
+                end
             end
         end
         
@@ -619,16 +772,16 @@ function analyze_peak_characteristics(msi_data::MSIData, instrument_analysis::Di
             results[:total_peaks_detected] = 0
         end
         
-        # For centroid data, FWHM is estimated from mass accuracy
+        # For centroid data, use much more permissive parameters
-        estimated_fwhm = estimated_mean_ppm_error * 2 # Centroid peaks are narrower, factor of 2-3 is common
+        # Don't estimate FWHM from mass accuracy - use reasonable defaults
-        results[:mean_fwhm_ppm] = estimated_fwhm
+        results[:mean_fwhm_ppm] = 50.0  # Reasonable default for centroid data
-        results[:median_fwhm_ppm] = estimated_fwhm # Defaulting median to mean
+        results[:median_fwhm_ppm] = 50.0
-        results[:mean_gaussian_r2] = 0.9 # Default for centroid, assuming good peak shapes
+        results[:mean_gaussian_r2] = 0.0  # Disable shape filtering for centroids
-        results[:peak_resolution_estimate] = 1e6 / estimated_fwhm
+        results[:peak_resolution_estimate] = 20000.0  # Reasonable estimate
         
         println("  - Mean peaks per spectrum: $(round(results[:mean_peaks_per_spectrum], digits=1))")
-        println("  - Estimated peak width: $(round(estimated_fwhm, digits=2)) ppm")
+        println("  - Using permissive FWHM for centroid data: 50.0 ppm")
-        println("  - Estimated resolution: $(round(results[:peak_resolution_estimate], digits=0))")
+        println("  - Shape filtering disabled for centroid data")
     end
     
     # Common prints
@@ -641,7 +794,16 @@ end
 """
     generate_preprocessing_recommendations(analysis_results::Dict)::Dict{Symbol, Any}
 
-Generates intelligent preprocessing recommendations based on the analysis results.
+Generates intelligent preprocessing recommendations by synthesizing the results from
+various analysis functions (`analyze_instrument_characteristics`, `analyze_signal_quality`, etc.).
+
+# Arguments
+- `analysis_results::Dict`: A dictionary containing the comprehensive analysis results from
+  the `run_preprocessing_analysis` pipeline.
+
+# Returns
+- `Dict{Symbol, Any}`: A dictionary where keys are preprocessing step names (e.g., `:smoothing`,
+  `:peak_picking`) and values are dictionaries of recommended parameters for that step.
 """
 function generate_preprocessing_recommendations(analysis_results::Dict)::Dict{Symbol, Any}
     recommendations = Dict{Symbol, Any}()
@@ -651,14 +813,17 @@ function generate_preprocessing_recommendations(analysis_results::Dict)::Dict{Sy
     mass_accuracy = get(analysis_results, :mass_accuracy, Dict())
     peak_analysis = get(analysis_results, :peak_analysis, Dict())
     
+    # Stabilization Recommendations
+    recommendations[:stabilization] = generate_stabilization_recommendations(signal_analysis)
+
     # Baseline Correction Recommendations
     recommendations[:baseline_correction] = generate_baseline_recommendations(inst_analysis, signal_analysis)
     
     # Smoothing Recommendations
     recommendations[:smoothing] = generate_smoothing_recommendations(inst_analysis, peak_analysis)
     
-    # Peak Picking Recommendations
+    # Peak Picking Recommendations - pass instrument analysis
-    recommendations[:peak_picking] = generate_peak_picking_recommendations(signal_analysis, peak_analysis)
+    recommendations[:peak_picking] = generate_peak_picking_recommendations(signal_analysis, peak_analysis, inst_analysis)
     
     # Normalization Recommendations
     recommendations[:normalization] = generate_normalization_recommendations(signal_analysis)
@@ -672,6 +837,7 @@ function generate_preprocessing_recommendations(analysis_results::Dict)::Dict{Sy
     return recommendations
 end
 
+"""Generate baseline correction recommendations based on data properties."""
 function generate_baseline_recommendations(inst_analysis, signal_analysis)
     recommendations = Dict{Symbol, Any}()
     
@@ -691,6 +857,7 @@ function generate_baseline_recommendations(inst_analysis, signal_analysis)
     return recommendations
 end
 
+"""Generate smoothing recommendations based on peak width and m/z step."""
 function generate_smoothing_recommendations(inst_analysis, peak_analysis)
     recommendations = Dict{Symbol, Any}()
     
@@ -713,19 +880,33 @@ function generate_smoothing_recommendations(inst_analysis, peak_analysis)
     return recommendations
 end
 
-function generate_peak_picking_recommendations(signal_analysis, peak_analysis)
+"""Generate peak picking recommendations from signal and peak analyses."""
+function generate_peak_picking_recommendations(signal_analysis, peak_analysis, inst_analysis)
     recommendations = Dict{Symbol, Any}()
     
+    acquisition_mode = get(inst_analysis, :acquisition_mode, :profile)
+    
+    if acquisition_mode == :centroid
+        # Much more permissive parameters for centroid data
+        recommendations[:snr_threshold] = 2.0  # Lower threshold for centroid
+        recommendations[:min_peak_width_ppm] = 0.0   # No minimum width for centroids
+        recommendations[:max_peak_width_ppm] = 200.0 # Very wide maximum for centroids
+        recommendations[:reason] = "Centroid data: using permissive parameters"
+    else
+        # Existing profile mode logic
         snr_threshold = get(signal_analysis, :suggested_snr, 3.0)
         fwhm_ppm = get(peak_analysis, :mean_fwhm_ppm, 20.0)
         
         recommendations[:snr_threshold] = snr_threshold
-    recommendations[:min_peak_width_ppm] = fwhm_ppm * 0.5  # Avoid detecting noise as peaks
+        recommendations[:min_peak_width_ppm] = fwhm_ppm * 0.5
-    recommendations[:max_peak_width_ppm] = fwhm_ppm * 3.0   # Avoid merging distinct peaks
+        recommendations[:max_peak_width_ppm] = fwhm_ppm * 3.0
+        recommendations[:reason] = "Profile data: using standard parameters"
+    end
     
     return recommendations
 end
 
+"""Generate normalization recommendations based on TIC variation."""
 function generate_normalization_recommendations(signal_analysis)
     recommendations = Dict{Symbol, Any}()
     
@@ -742,6 +923,7 @@ function generate_normalization_recommendations(signal_analysis)
     return recommendations
 end
 
+"""Generate alignment recommendations based on calibration status and mass error."""
 function generate_alignment_recommendations(mass_accuracy, inst_analysis)
     recommendations = Dict{Symbol, Any}()
     
@@ -765,6 +947,7 @@ function generate_alignment_recommendations(mass_accuracy, inst_analysis)
     return recommendations
 end
 
+"""Generate binning recommendations based on peak width and mass accuracy."""
 function generate_binning_recommendations(peak_analysis, mass_accuracy)
     recommendations = Dict{Symbol, Any}()
     
@@ -794,18 +977,47 @@ function generate_binning_recommendations(peak_analysis, mass_accuracy)
     return recommendations
 end
 
+"""Generate intensity stabilization recommendations."""
+function generate_stabilization_recommendations(signal_analysis)
+    recommendations = Dict{Symbol, Any}()
+    # Default to sqrt, as it's a common and generally robust transformation.
+    # More advanced logic could analyze intensity distribution skewness if needed.
+    recommendations[:method] = :sqrt 
+    return recommendations
+end
+
 # =============================================================================
 # Pre-Analysis Pipeline for Auto Parameter Determination
 # =============================================================================
 
 """
-    run_preprocessing_analysis(msi_data::MSIData; 
+    run_preprocessing_analysis(msi_data; ...)
-                              reference_peaks::Dict{Float64, String}=Dict(),
-                              region_masks::Dict=Dict(),
-                              sample_size::Int=100)::Dict{Symbol, Any}
 
 Runs a comprehensive pre-analysis pipeline to determine optimal preprocessing parameters.
-This function analyzes the dataset to provide intelligent defaults for preprocessing steps.
+This function orchestrates a series of analysis steps on a sample of the dataset to
+provide intelligent defaults for a full preprocessing workflow.
+
+The pipeline consists of several phases:
+1.  **Instrument & Data Characteristics**: Infers acquisition mode, m/z axis type, etc.
+2.  **Noise & Signal Quality**: Estimates noise, SNR, and TIC variation.
+3.  **Mass Accuracy**: Calculates PPM error against reference peaks (if provided).
+4.  **Spatial Regions**: Analyzes regional variations (if masks are provided).
+5.  **Peak Characteristics**: Measures peak width, shape, and density.
+6.  **Recommendations**: Synthesizes all analysis results into actionable parameter suggestions.
+
+# Arguments
+- `msi_data::MSIData`: The main MSI data object.
+- `reference_peaks::Dict`: Optional. Known m/z values for mass accuracy analysis.
+- `region_masks::Dict`: Optional. Named `BitMatrix` masks for regional analysis.
+- `sample_size::Int`: The number of spectra to sample for the analysis.
+- `mask_path::String`: Optional path to a PNG mask to restrict analysis to a specific ROI.
+- `spectrum_indices::AbstractVector{Int}`: Optional vector of indices to restrict analysis to,
+  overriding `mask_path` and `sample_size` for selection.
+
+# Returns
+- `Dict{Symbol, Any}`: A nested dictionary containing the results of each analysis phase
+  and a final `:recommendations` dictionary. The recommendations are also stored in
+  `msi_data.preprocessing_hints`.
 """
 function run_preprocessing_analysis(msi_data::MSIData; 
                                    reference_peaks::Dict{Float64, String}=Dict{Float64, String}(),
@@ -1019,7 +1231,7 @@ function calculate_resolution_fwhm(mz::Real, profile_mz::AbstractVector{<:Real},
     return fwhm > 0 ? Float64(mz) / fwhm : NaN
 end
 
-# Helper functions for FWHM calculation
+"""Helper to find the last index in a vector with a value below a threshold, used for FWHM."""
 function find_last_below(v::AbstractVector{<:Real}, threshold::Real)
     for i in length(v):-1:2
         if v[i] >= threshold && v[i-1] < threshold
@@ -1029,6 +1241,7 @@ function find_last_below(v::AbstractVector{<:Real}, threshold::Real)
     return 0
 end
 
+"""Helper to find the first index in a vector with a value below a threshold, used for FWHM."""
 function find_first_below(v::AbstractVector{<:Real}, threshold::Real)
     for i in 1:(length(v)-1)
         if v[i] >= threshold && v[i+1] < threshold
@@ -1101,18 +1314,16 @@ function _fit_gaussian_and_r2(mz::AbstractVector{<:Real}, intensity::AbstractVec
     end_idx = min(n, peak_idx + half_window)
     
     # Ensure there's enough data to fit
-    if (end_idx - start_idx + 1) < 3
+    count = end_idx - start_idx + 1
+    if count < 3
         return 0.0
     end
 
-    x_data = mz[start_idx:end_idx]
-    y_data = intensity[start_idx:end_idx]
-
     # Estimate Gaussian parameters
     # Amplitude (A): peak intensity
-    A_est = intensity[peak_idx]
+    A_est = float(intensity[peak_idx])
     # Mean (μ): m/z at peak intensity
-    mu_est = mz[peak_idx]
+    mu_est = float(mz[peak_idx])
     # Standard deviation (σ): related to FWHM. FWHM = 2 * sqrt(2 * ln(2)) * σ ≈ 2.355 * σ
     # So, σ ≈ FWHM / 2.355
     fwhm_delta_m = _calculate_fwhm_delta_m(mz, intensity, peak_idx)
@@ -1126,19 +1337,27 @@ function _fit_gaussian_and_r2(mz::AbstractVector{<:Real}, intensity::AbstractVec
         return 0.0
     end
 
-    # Gaussian function
+    # Calculate SS_res, mean_y in a single pass to avoid allocations
-    gaussian(x, A, mu, sigma) = A * exp.(-(x .- mu).^2 ./ (2 * sigma^2))
+    SS_res = 0.0
+    sum_y = 0.0
     
-    # Generate estimated Gaussian curve
+    @inbounds for i in start_idx:end_idx
-    y_est = gaussian(x_data, A_est, mu_est, sigma_est)
+        x_val = float(mz[i])
+        y_val = float(intensity[i])
         
-    # Calculate pseudo R-squared
+        # Gaussian function estimate
-    # R^2 = 1 - (SS_res / SS_tot)
+        y_est = A_est * exp(-((x_val - mu_est)^2) / (2 * sigma_est^2))
-    # SS_res = sum((y_data - y_est).^2)
-    # SS_tot = sum((y_data - mean(y_data)).^2)
         
-    SS_res = sum((y_data .- y_est).^2)
+        SS_res += (y_val - y_est)^2
-    SS_tot = sum((y_data .- mean(y_data)).^2)
+        sum_y += y_val
+    end
+
+    mean_y = sum_y / count
+    SS_tot = 0.0
+    
+    @inbounds for i in start_idx:end_idx
+        SS_tot += (float(intensity[i]) - mean_y)^2
+    end
 
     if SS_tot == 0
         return 1.0 # Perfect fit if all y_data are the same
@@ -1270,10 +1489,11 @@ function main_precalculation(msi_data::MSIData;
     recs = get(analysis_results, :recommendations, Dict())
     signal_analysis = get(analysis_results, :signal_analysis, Dict())
     peak_analysis = get(analysis_results, :peak_analysis, Dict())
-    mass_accuracy = get(analysis_results, :mass_accuracy, Dict())
+    mass_accuracy = get(analysis_results, :mass_accuracy, nothing) # Can be nothing
     inst_analysis = get(analysis_results, :instrument_analysis, Dict())
 
     # Initialize parameter dictionaries for each step
+    stab_params = Dict{Symbol, Any}()
     cal_params = Dict{Symbol, Any}()
     sm_params = Dict{Symbol, Any}()
     bc_params = Dict{Symbol, Any}()
@@ -1284,11 +1504,22 @@ function main_precalculation(msi_data::MSIData;
     pb_params = Dict{Symbol, Any}()
 
     # --- Populate Parameters for each step ---
+    # Stabilization
+    if !isempty(recs) && haskey(recs, :stabilization)
+        stab_rec = recs[:stabilization]
+        stab_params[:method] = get(stab_rec, :method, :sqrt)
+    else
+        stab_params[:method] = :sqrt # Default
+    end
 
     # Calibration & Alignment (Note: These are intertwined in the current logic)
     calibration_required = false
+    mean_ppm_error = NaN
+    suggested_bin_tol = NaN
+    if mass_accuracy !== nothing
         mean_ppm_error = get(get(mass_accuracy, :global_accuracy, Dict()), :mean_ppm_error, NaN)
         suggested_bin_tol = get(mass_accuracy, :suggested_bin_tolerance, NaN)
+    end
     
     if !isempty(recs) && haskey(recs, :alignment) && get(recs[:alignment], :required, false)
         calibration_required = true
@@ -1400,30 +1631,46 @@ function main_precalculation(msi_data::MSIData;
     end
 
     # Peak Picking
-    # Robust method selection with fallback
+    # Robust method selection based on acquisition mode
-    acquisition_mode = get(inst_analysis, :acquisition_mode, :profile) # Default to profile
+    acquisition_mode = get(inst_analysis, :acquisition_mode, :unknown)
     pp_params[:method] = acquisition_mode == :profile ? :profile : :centroid
 
+    if acquisition_mode == :centroid
+        # Much more permissive parameters for centroid data
+        pp_params[:snr_threshold] = 2.0  # Lower for centroid
+        pp_params[:min_peak_width_ppm] = 0.0    # No minimum width
+        pp_params[:max_peak_width_ppm] = 200.0  # Very wide maximum
+        pp_params[:min_peak_shape_r2] = 0.0     # Disable shape filtering
+        
+        # Much lower prominence threshold for centroid
+        estimated_noise = get(signal_analysis, :noise_mean, NaN)
+        if isfinite(estimated_noise)
+            pp_params[:min_peak_prominence] = max(estimated_noise * 0.5, 0.001)  # Much lower
+        else
+            pp_params[:min_peak_prominence] = 0.001  # Very permissive
+        end
+        
+        pp_params[:half_window] = 2  # Smaller window for centroid data
+        pp_params[:merge_peaks_tolerance] = 10.0  # More permissive merging
+    else # Profile mode logic
         if !isempty(recs) && haskey(recs, :peak_picking)
             pk_rec = recs[:peak_picking]
-        pp_params[:snr_threshold] = get(pk_rec, :snr_threshold, 3.0) # Default to 3.0
+            pp_params[:snr_threshold] = get(pk_rec, :snr_threshold, 2.0) # Default to 2.0
             pp_params[:min_peak_width_ppm] = get(pk_rec, :min_peak_width_ppm, nothing)
             pp_params[:max_peak_width_ppm] = get(pk_rec, :max_peak_width_ppm, nothing)
         else
-        pp_params[:snr_threshold] = 3.0 # Default to 3.0
+            pp_params[:snr_threshold] = 2.0 # Default to 2.0
             pp_params[:min_peak_width_ppm] = nothing
             pp_params[:max_peak_width_ppm] = nothing
         end
         
-    # Robust prominence calculation with safety cap
+        # Robust prominence calculation with safety cap for profile
         estimated_noise = get(signal_analysis, :noise_mean, NaN)
         if isfinite(estimated_noise)
-        # Never be more aggressive than 0.05, a reasonable upper limit
             calculated_prominence = round(estimated_noise * 2, digits=4)
-        pp_params[:min_peak_prominence] = min(calculated_prominence, 0.05)
+            pp_params[:min_peak_prominence] = min(calculated_prominence, 0.005) 
         else
-        # Fallback to a safe, non-aggressive value if noise couldn't be estimated
+            pp_params[:min_peak_prominence] = 0.005 
-        pp_params[:min_peak_prominence] = 0.01
         end
 
         if isfinite(suggested_bin_tol)
@@ -1432,31 +1679,38 @@ function main_precalculation(msi_data::MSIData;
             pp_params[:merge_peaks_tolerance] = nothing
         end
 
-    # Robust half_window calculation with safety floor
+        # Robust half_window calculation with safety floor for profile
         mean_fwhm_ppm = get(peak_analysis, :mean_fwhm_ppm, NaN)
         avg_mz_step = get(inst_analysis, :average_mz_step, NaN)
         if isfinite(mean_fwhm_ppm) && isfinite(avg_mz_step) && avg_mz_step > 0
             fwhm_mz = 500.0 * mean_fwhm_ppm / 1e6 # At typical m/z 500
             window_points = fwhm_mz / avg_mz_step
             calculated_half_window = ceil(Int, window_points / 2)
-        # Ensure half_window is at least 3, a safe absolute minimum
             pp_params[:half_window] = max(calculated_half_window, 3)
         else
-        # Fallback if calculation is not possible
             pp_params[:half_window] = 5
         end
         
-    # Robust R^2 calculation with floor
+        # Robust R^2 calculation with floor for profile
         mean_r2 = get(peak_analysis, :mean_gaussian_r2, NaN)
         if isfinite(mean_r2)
-        pp_params[:min_peak_shape_r2] = round(max(0.5, mean_r2 * 0.8), digits=2)
+            pp_params[:min_peak_shape_r2] = round(max(0.0, mean_r2 * 0.8), digits=2)
         else
-        # Fallback to a reasonable default
+            pp_params[:min_peak_shape_r2] = 0.0
-        pp_params[:min_peak_shape_r2] = 0.6
+        end
     end
 
 
     # Peak Selection
+    if acquisition_mode == :centroid
+        # Much more permissive parameters for centroid data
+        ps_params[:min_snr] = 1.5  # Even lower than picking threshold
+        ps_params[:min_fwhm_ppm] = 0.0      # No minimum
+        ps_params[:max_fwhm_ppm] = 500.0    # Very wide maximum
+        ps_params[:min_shape_r2] = 0.0      # Disable shape filtering
+        ps_params[:frequency_threshold] = nothing # User input dependent / Hard to determine
+        ps_params[:correlation_threshold] = nothing # Hard to determine
+    else # Profile mode logic
         if haskey(pp_params, :snr_threshold) && pp_params[:snr_threshold] !== nothing
             ps_params[:min_snr] = pp_params[:snr_threshold]
         else
@@ -1471,18 +1725,20 @@ function main_precalculation(msi_data::MSIData;
                 ps_params[:min_fwhm_ppm] = nothing
                 ps_params[:max_fwhm_ppm] = nothing
             end
+            mean_r2 = get(peak_analysis, :mean_gaussian_r2, NaN)
             if isfinite(mean_r2)
-            ps_params[:min_shape_r2] = round(max(0.5, mean_r2 * 0.8), digits=2)
+                ps_params[:min_shape_r2] = round(max(0.0, mean_r2 * 0.8), digits=2)
             else
-            ps_params[:min_shape_r2] = nothing
+                ps_params[:min_shape_r2] = 0.0 # Adjusted fallback
             end
         else
             ps_params[:min_fwhm_ppm] = nothing
             ps_params[:max_fwhm_ppm] = nothing
             ps_params[:min_shape_r2] = nothing
         end
-    ps_params[:frequency_threshold] = nothing # User input dependent / Hard to determine
+        ps_params[:frequency_threshold] = nothing 
-    ps_params[:correlation_threshold] = nothing # Hard to determine
+        ps_params[:correlation_threshold] = nothing 
+    end
 
 
     # Peak Binning
@@ -1525,6 +1781,7 @@ function main_precalculation(msi_data::MSIData;
     end
 
     return Dict(
+        :Stabilization => stab_params,
         :Calibration => cal_params,
         :Smoothing => sm_params,
         :BaselineCorrection => bc_params,
@@ -1532,6 +1789,6 @@ function main_precalculation(msi_data::MSIData;
         :PeakPicking => pp_params,
         :PeakAlignment => pa_params,
         :PeakSelection => ps_params,
-        :PeakBinningParams => pb_params
+        :PeakBinning => pb_params
     )
 end

src/Preprocessing.jl

@@ -12,6 +12,7 @@ generation.
 # =============================================================================
 
 using Statistics  # For mean, median
+using SparseArrays
 using StatsBase   # For mad (Median Absolute Deviation)
 using SavitzkyGolay # For SavitzkyGolay filtering
 using Dates         # For now()
@@ -21,6 +22,7 @@ using ContinuousWavelets # For CWT peak detection
 using ImageFiltering # For localmaxima in detect_peaks_wavelet
 using Interpolations # For calibration
 using Loess # For robust peak alignment
+using Base.Threads # For multithreading in apply functions
 
 # =============================================================================
 # Data Structures
@@ -30,17 +32,33 @@ using Loess # For robust peak alignment
     FeatureMatrix
 
 A struct to hold the final feature matrix generated from the preprocessing pipeline.
+
+# Fields
+- `matrix::Array{Float64,2}`: The feature matrix where rows correspond to samples (spectra)
+  and columns correspond to features (m/z bins).
+- `mz_bins::Vector{Tuple{Float64,Float64}}`: A vector of tuples defining the start and
+  end m/z for each bin (column) in the `matrix`.
+- `sample_ids::Vector{Int}`: A vector of identifiers for each sample (row) in the `matrix`.
 """
 struct FeatureMatrix
-    matrix::Array{Float64,2}
+    matrix::AbstractMatrix{Float64}
     mz_bins::Vector{Tuple{Float64,Float64}}
     sample_ids::Vector{Int}
 end
 
 """
+    MutableSpectrum
+
 A mutable struct to hold spectrum data. Using a mutable struct allows
 in-place modification of fields (like intensity or m/z), which dramatically
 reduces memory allocations compared to creating new immutable tuples at each step.
+
+# Fields
+- `id::Int`: A unique identifier for the spectrum.
+- `mz::AbstractVector{Float64}`: The m/z values of the spectrum.
+- `intensity::AbstractVector{Float64}`: The intensity values corresponding to the m/z values.
+- `peaks::Vector{NamedTuple}`: A vector of detected peaks, each a `NamedTuple` with fields
+  like `:mz`, `:intensity`, `:fwhm`, etc.
 """
 mutable struct MutableSpectrum
     id::Int
@@ -62,8 +80,18 @@ abstract type AbstractPreprocessingStep end
 """
     Calibration(; method=:internal_standards, ...)
 
-A preprocessing step for mass calibration. Set a parameter to `nothing` to use an
+A preprocessing step for mass calibration. Corrects systematic mass errors in the m/z axis.
-auto-determined value from the data where applicable.
+Set a parameter to `nothing` to use an auto-determined value from the data where applicable.
+
+# Arguments
+- `method::Symbol`: The calibration method. Currently supports `:internal_standards`.
+- `internal_standards::Union{Dict{Float64, String}, Nothing}`: A dictionary mapping theoretical
+  m/z values of internal standards to their names.
+- `base_peak_mz_references::Union{Vector{Float64}, Nothing}`: A vector of reference m/z values
+  for base peak calibration (not yet implemented).
+- `ppm_tolerance::Union{Float64, Nothing}`: The tolerance in parts-per-million (ppm) for matching
+  peaks to internal standards.
+- `fit_order::Int`: The polynomial order for the calibration fit (e.g., 1 for linear, 2 for quadratic).
 """
 struct Calibration <: AbstractPreprocessingStep
     method::Symbol
@@ -80,7 +108,16 @@ end
 """
     BaselineCorrection(; method=:snip, ...)
 
-A preprocessing step for baseline correction.
+A preprocessing step for baseline correction. This step estimates and subtracts the
+background noise (baseline) from the spectral intensities.
+
+# Arguments
+- `method::Symbol`: The algorithm to use. Options include `:snip` (Sensitive Nonlinear
+  Iterative Peak clipping), `:convex_hull`, and `:median`.
+- `iterations::Union{Int, Nothing}`: The number of iterations for the SNIP algorithm.
+  A higher number results in a more aggressive baseline.
+- `window::Union{Int, Nothing}`: The window size for the `:median` method, determining
+  the local region for median calculation.
 """
 struct BaselineCorrection <: AbstractPreprocessingStep
     method::Symbol
@@ -95,7 +132,15 @@ end
 """
     Smoothing(; method=:savitzky_golay, ...)
 
-A preprocessing step for spectral smoothing.
+A preprocessing step for spectral smoothing. This helps to reduce high-frequency noise
+in the intensity data.
+
+# Arguments
+- `method::Symbol`: The smoothing algorithm. Options are `:savitzky_golay` and `:moving_average`.
+- `window::Union{Int, Nothing}`: The size of the smoothing window. For Savitzky-Golay,
+  this must be an odd integer.
+- `order::Union{Int, Nothing}`: The polynomial order for the Savitzky-Golay filter. Must be
+  less than the window size.
 """
 struct Smoothing <: AbstractPreprocessingStep
     method::Symbol
@@ -112,7 +157,15 @@ end
 """
     Normalization(; method=:tic)
 
-A preprocessing step for intensity normalization.
+A preprocessing step for intensity normalization. This corrects for variations in total
+ion current between different spectra, making them more comparable.
+
+# Arguments
+- `method::Symbol`: The normalization method. Options include:
+  - `:tic`: Total Ion Current normalization (divides by the sum of intensities).
+  - `:median`: Divides by the median intensity.
+  - `:rms`: Root Mean Square normalization.
+  - `:none`: No normalization is applied.
 """
 struct Normalization <: AbstractPreprocessingStep
     method::Symbol
@@ -125,7 +178,25 @@ end
 """
     PeakPicking(; method=nothing, ...)
 
-A preprocessing step for peak detection.
+A preprocessing step for peak detection. This step identifies peaks (signals of interest)
+in the profile or centroided spectra.
+
+# Arguments
+- `method::Union{Symbol, Nothing}`: The peak detection algorithm.
+  - `:profile`: For profile-mode data, using local maxima and quality filters.
+  - `:wavelet`: Continuous Wavelet Transform (CWT) based peak detection.
+  - `:centroid`: For centroid-mode data, essentially a filtering step.
+- `snr_threshold::Union{Float64, Nothing}`: Signal-to-Noise Ratio threshold. Peaks with SNR
+  below this value are discarded.
+- `half_window::Union{Int, Nothing}`: The number of data points to the left and right of a
+  potential peak to consider for local maximum detection (for `:profile`).
+- `min_peak_prominence::Union{Float64, Nothing}`: The minimum required prominence of a peak,
+  expressed as a fraction of its height.
+- `merge_peaks_tolerance::Union{Float64, Nothing}`: The m/z tolerance within which to merge
+  adjacent peaks, keeping the more intense one.
+- `min_peak_width_ppm, max_peak_width_ppm`: Minimum and maximum acceptable peak width (FWHM) in ppm.
+- `min_peak_shape_r2`: Minimum R-squared value from a Gaussian fit to the peak, used as a
+  quality measure for peak shape.
 """
 struct PeakPicking <: AbstractPreprocessingStep
     method::Union{Symbol, Nothing} # :profile, :wavelet, :centroid
@@ -145,7 +216,20 @@ end
 """
     PeakAlignment(; method=:lowess, ...)
 
-A preprocessing step for peak alignment.
+A preprocessing step for peak alignment. This corrects for m/z shifts between spectra,
+ensuring that the same analyte peak appears at the same m/z across all samples.
+
+# Arguments
+- `method::Symbol`: The alignment algorithm. Options: `:lowess`, `:linear`, `:ransac`.
+- `span::Union{Float64, Nothing}`: The span parameter for LOWESS regression, controlling smoothness.
+- `tolerance::Union{Float64, Nothing}`: The tolerance for matching peaks between the target
+  and reference spectrum.
+- `tolerance_unit::Union{Symbol, Nothing}`: The unit for `tolerance`, either `:mz` (absolute)
+  or `:ppm` (relative).
+- `max_shift_ppm::Union{Float64, Nothing}`: The maximum allowed m/z shift in ppm to prevent
+  spurious peak matches.
+- `min_matched_peaks::Union{Int, Nothing}`: The minimum number of matching peaks required
+  to perform the alignment.
 """
 struct PeakAlignment <: AbstractPreprocessingStep
     method::Symbol
@@ -163,7 +247,19 @@ end
 """
     PeakSelection(; frequency_threshold=nothing, ...)
 
-A preprocessing step for peak selection (filtering).
+A preprocessing step for peak selection (filtering). After peak detection, this step
+filters the detected peaks based on various quality criteria to remove noise and
+irrelevant signals.
+
+# Arguments
+- `frequency_threshold::Union{Float64, Nothing}`: The minimum fraction of spectra in which a
+  peak must be present to be kept.
+- `min_snr::Union{Float64, Nothing}`: Minimum Signal-to-Noise Ratio.
+- `min_fwhm_ppm, max_fwhm_ppm`: Minimum and maximum Full Width at Half Maximum in ppm.
+- `min_shape_r2::Union{Float64, Nothing}`: Minimum R-squared value from a Gaussian fit,
+  filtering for good peak shape.
+- `correlation_threshold::Union{Float64, Nothing}`: Minimum correlation with neighboring
+  peaks (not yet implemented).
 """
 struct PeakSelection <: AbstractPreprocessingStep
     frequency_threshold::Union{Float64, Nothing}
@@ -179,11 +275,24 @@ struct PeakSelection <: AbstractPreprocessingStep
 end
 
 """
-    PeakBinningParams(; method=:adaptive, ...)
+    PeakBinning(; method=:adaptive, ...)
 
-A preprocessing step for peak binning.
+A preprocessing step for peak binning. This step groups peaks from all spectra into
+common m/z bins to generate a feature matrix.
+
+# Arguments
+- `method::Symbol`: The binning strategy.
+  - `:adaptive`: Creates bins based on the density of detected peaks.
+  - `:uniform`: Creates a fixed number of equally spaced bins over the m/z range.
+- `tolerance, tolerance_unit`: Tolerance for grouping peaks into a bin in `:adaptive` mode.
+- `frequency_threshold`: The minimum fraction of spectra a bin must contain a peak in to be kept.
+- `min_peak_per_bin`: The minimum number of individual peaks required to form a bin in `:adaptive` mode.
+- `max_bin_width_ppm`: Maximum width of a bin in ppm for `:adaptive` mode.
+- `intensity_weighted_centers`: If `true`, calculates bin centers as an intensity-weighted
+  average of the peaks within it.
+- `num_uniform_bins`: The number of bins to create for the `:uniform` method.
 """
-struct PeakBinningParams <: AbstractPreprocessingStep
+struct PeakBinning <: AbstractPreprocessingStep
     method::Symbol
     tolerance::Union{Float64, Nothing}
     tolerance_unit::Union{Symbol, Nothing}
@@ -193,7 +302,7 @@ struct PeakBinningParams <: AbstractPreprocessingStep
     intensity_weighted_centers::Bool
     num_uniform_bins::Union{Int, Nothing}
 
-    function PeakBinningParams(; method=:adaptive, tolerance=nothing, tolerance_unit=nothing, frequency_threshold=nothing, min_peak_per_bin=nothing, max_bin_width_ppm=nothing, intensity_weighted_centers=true, num_uniform_bins=nothing)
+    function PeakBinning(; method=:adaptive, tolerance=nothing, tolerance_unit=nothing, frequency_threshold=nothing, min_peak_per_bin=nothing, max_bin_width_ppm=nothing, intensity_weighted_centers=true, num_uniform_bins=nothing)
         new(method, tolerance, tolerance_unit, frequency_threshold, min_peak_per_bin, max_bin_width_ppm, intensity_weighted_centers, num_uniform_bins)
     end
 end
@@ -237,7 +346,7 @@ Checks include:
 - `mz` and `intensity` have the same length.
 - All `m/z` values are finite and non-negative.
 - All `intensity` values are finite and non-negative.
-- `m/z` values are strictly increasing (no duplicates or decreasing values).
+- `m/z` values are monotonically non-decreasing.
 """
 function validate_spectrum(mz::AbstractVector{<:Real}, intensity::AbstractVector{<:Real})::Bool
     # 1. Check for empty vectors
@@ -278,11 +387,22 @@ end
 # =============================================================================
 
 """
-    transform_intensity(intensity; method=:sqrt) -> Vector
+    transform_intensity_core(intensity; method=:sqrt) -> Vector
 
-Applies a variance-stabilizing transformation to the intensity vector.
+Applies a variance-stabilizing transformation to the intensity vector. This can
+help to make the variance of the signal more constant across the intensity range,
+which is often an assumption of downstream statistical methods.
+
+# Arguments
+- `intensity::AbstractVector{<:Real}`: The input intensity values.
+- `method::Symbol`: The transformation to apply. Options are:
+  - `:sqrt`: Square root transformation.
+  - `:log`: Natural log transformation.
+  - `:log2`: Base-2 log transformation.
+  - `:log10`: Base-10 log transformation.
+  - `:log1p`: Natural log of `1 + x`, useful for data with zeros.
 """
-function transform_intensity(intensity::AbstractVector{<:Real}; method::Symbol=:sqrt)
+function transform_intensity_core(intensity::AbstractVector{<:Real}; method::Symbol=:sqrt)
     if method === :sqrt
         return sqrt.(max.(zero(eltype(intensity)), intensity))
     elseif method === :log1p
@@ -299,7 +419,7 @@ function transform_intensity(intensity::AbstractVector{<:Real}; method::Symbol=:
 end
 
 """
-    smooth_spectrum(y::AbstractVector{<:Real}; method::Symbol=:savitzky_golay, window::Int=9, order::Int=2) -> Vector
+    smooth_spectrum_core(y::AbstractVector{<:Real}; method::Symbol=:savitzky_golay, window::Int=9, order::Int=2) -> Vector
 
 Applies a smoothing filter to the intensity data.
 
@@ -309,7 +429,7 @@ Applies a smoothing filter to the intensity data.
 - `window`: The window size for the filter.
 - `order`: The polynomial order for Savitzky-Golay
 """
-function smooth_spectrum(y::AbstractVector{<:Real}; method::Symbol=:savitzky_golay, window::Int=9, order::Int=2)
+function smooth_spectrum_core(y::AbstractVector{<:Real}; method::Symbol=:savitzky_golay, window::Int=9, order::Int=2)
     if window < 3
         throw(ArgumentError("Window size must be at least 3"))
     end
@@ -368,38 +488,31 @@ Estimates the baseline of a spectrum using the SNIP algorithm (internal implemen
 function _snip_baseline_impl(y::AbstractVector{<:Real}; iterations::Int=100)
     n = length(y)
     
-    # Initialize two buffers. b1 holds the current baseline estimate, b2 for the next.
+    # Initialize the baseline estimate array once
-    # Always convert to Float64 to ensure type stability and avoid copying if already correct type
     b1 = collect(float.(y))
-    b2 = similar(b1)        
-    
-    current_b = b1
-    next_b = b2
     
     for k in 1:iterations
-        # Calculate next baseline estimate into `next_b` based on `current_b`
+        prev_val = b1[1]
-        # Boundary conditions
+        b1[1] = min(b1[1], b1[2])
-        if n > 1
-            next_b[1] = min(current_b[1], current_b[2])
-            next_b[n] = min(current_b[n], current_b[n-1])
-        end
         
         @inbounds for i in 2:n-1
-            next_b[i] = min(current_b[i], 0.5 * (current_b[i-1] + current_b[i+1]))
+            curr_val = b1[i]
+            b1[i] = min(curr_val, 0.5 * (prev_val + b1[i+1]))
+            prev_val = curr_val
+        end
+        b1[n] = min(b1[n], prev_val)
     end
     
-        # Swap references for the next iteration (no data copy here)
+    # Return the final baseline estimate
-        current_b, next_b = next_b, current_b 
+    return b1
-    end
-    
-    # Return the final baseline estimate (which is in current_b after the last swap)
-    return current_b
 end
 
 """
     convex_hull_baseline(y) -> Vector
 
-Estimates the baseline of a spectrum using the convex hull algorithm.
+Estimates the baseline of a spectrum using the convex hull algorithm. This method
+finds the lower convex hull of the spectrum, which is then used as the baseline.
+It is generally faster than SNIP but can be less flexible.
 """
 function convex_hull_baseline(y::AbstractVector{<:Real})
     n = length(y)
@@ -455,7 +568,7 @@ function median_baseline(y::AbstractVector{<:Real}; window::Int=20)
 end
 
 """
-    apply_baseline_correction(y::AbstractVector{<:Real}; method::Symbol=:snip, iterations::Int=100, window::Int=20) -> Vector
+    apply_baseline_correction_core(y::AbstractVector{<:Real}; method::Symbol=:snip, iterations::Int=100, window::Int=20) -> Vector
 
 Applies a baseline correction algorithm to the intensity data.
 
@@ -465,7 +578,7 @@ Applies a baseline correction algorithm to the intensity data.
 - `iterations`: Iterations for SNIP method.
 - `window`: Window size for median method.
 """
-function apply_baseline_correction(y::AbstractVector{<:Real}; method::Symbol=:snip, iterations::Int=100, window::Int=20)
+function apply_baseline_correction_core(y::AbstractVector{<:Real}; method::Symbol=:snip, iterations::Int=100, window::Int=20)
     if method === :snip
         return _snip_baseline_impl(y, iterations=iterations)
     elseif method === :convex_hull
@@ -485,7 +598,9 @@ end
 """
     tic_normalize(y) -> Vector
 
-Normalizes spectrum intensities to the Total Ion Current (TIC).
+Normalizes spectrum intensities to the Total Ion Current (TIC). Each intensity value
+is divided by the sum of all intensities in the spectrum. This method assumes that the
+total number of ions produced is similar for all samples.
 """
 function tic_normalize(y::AbstractVector{<:Real})
     s = sum(y)
@@ -496,6 +611,18 @@ end
     pqn_normalize(M) -> Matrix
 
 Performs Probabilistic Quotient Normalization (PQN) on a matrix of spectra.
+This is a more robust normalization method that is less sensitive to a small
+number of highly abundant, variable peaks compared to TIC.
+
+# Steps:
+1. A reference spectrum is calculated (typically the median spectrum across all samples).
+2. For each spectrum, the quotients of its intensities and the reference spectrum's
+   intensities are calculated.
+3. The median of these quotients is found for each spectrum.
+4. Each spectrum is divided by its median quotient.
+
+# Arguments
+- `M::AbstractMatrix{<:Real}`: A matrix where columns are spectra and rows are m/z bins.
 """
 function pqn_normalize(M::AbstractMatrix{<:Real})
     M_float = collect(float.(M))
@@ -532,7 +659,7 @@ function rms_normalize(y::AbstractVector{<:Real})
 end
 
 """
-    apply_normalization(y::AbstractVector{<:Real}; method::Symbol=:tic) -> Vector
+    apply_normalization_core(y::AbstractVector{<:Real}; method::Symbol=:tic) -> Vector
 
 Applies a per-spectrum normalization algorithm to the intensity data.
 
@@ -540,7 +667,7 @@ Applies a per-spectrum normalization algorithm to the intensity data.
 - `y`: The intensity data.
 - `method`: The normalization method (:tic, :median, :rms, or :none).
 """
-function apply_normalization(y::AbstractVector{<:Real}; method::Symbol=:tic)
+function apply_normalization_core(y::AbstractVector{<:Real}; method::Symbol=:tic)::Vector
     if method === :tic
         return tic_normalize(y)
     elseif method === :median
@@ -555,188 +682,12 @@ function apply_normalization(y::AbstractVector{<:Real}; method::Symbol=:tic)
     end
 end
 
-"""
-    remove_matrix_peaks_from_spectrum(mz::AbstractVector{<:Real}, intensity::AbstractVector{<:Real},
-                                      matrix_peak_mzs::AbstractVector{<:Real};
-                                      tolerance::Float64=0.002, tolerance_unit::Symbol=:mz,
-                                      removal_method::Symbol=:zero_out) -> Vector{Float64}
-
-Removes or reduces intensity around specified matrix peaks in a single spectrum.
-
-# Arguments
-- `mz`: The m/z vector of the spectrum.
-- `intensity`: The intensity vector of the spectrum.
-- `matrix_peak_mzs`: A list of m/z values identified as matrix peaks.
-- `tolerance`: The m/z tolerance for matching matrix peaks.
-- `tolerance_unit`: Unit of tolerance (:mz or :ppm).
-- `removal_method`: How to remove the peaks (:zero_out or :subtract).
-
-# Returns
-- `Vector{Float64}`: The modified intensity vector.
-"""
-function remove_matrix_peaks_from_spectrum(mz::AbstractVector{<:Real}, intensity::AbstractVector{<:Real},
-                                           matrix_peak_mzs::AbstractVector{<:Real};
-                                           tolerance::Float64=0.002, tolerance_unit::Symbol=:mz,
-                                           removal_method::Symbol=:zero_out)
-    modified_intensity = copy(intensity)
-    
-    for matrix_mz in matrix_peak_mzs
-        # Calculate dynamic tolerance if in PPM
-        current_tolerance = (tolerance_unit == :ppm) ? (matrix_mz * tolerance / 1e6) : tolerance
-        
-        # Find indices within the tolerance window
-        indices_to_modify = findall(m -> abs(m - matrix_mz) <= current_tolerance, mz)
-        
-        if !isempty(indices_to_modify)
-            if removal_method == :zero_out
-                modified_intensity[indices_to_modify] .= 0.0
-            elseif removal_method == :subtract
-                # Baseline-aware subtraction: replace peak with a line connecting its "feet"
-                idx_start = indices_to_modify[1]
-                idx_end = indices_to_modify[end]
-                
-                # Ensure we are not at the very edge of the spectrum
-                if idx_start > 1 && idx_end < length(modified_intensity)
-                    y1 = modified_intensity[idx_start - 1]
-                    y2 = modified_intensity[idx_end + 1]
-                    x1 = idx_start - 1
-                    x2 = idx_end + 1
-
-                    # Linearly interpolate the baseline under the peak
-                    for i in idx_start:idx_end
-                        # y = y1 + (y2 - y1) * (x - x1) / (x2 - x1)
-                        baseline_val = y1 + (y2 - y1) * (i - x1) / (x2 - x1)
-                        # Set the intensity to the baseline, but don't increase it (e.g., if baseline is above signal)
-                        modified_intensity[i] = min(modified_intensity[i], baseline_val)
-                    end
-                else
-                    # If peak is at the edge, we can't interpolate, so just zero it out
-                    modified_intensity[indices_to_modify] .= 0.0
-                end
-            else
-                @warn "Unsupported matrix peak removal method: $removal_method. Skipping."
-            end
-        end
-    end
-    return modified_intensity
-end
-
-function identify_matrix_peaks_from_blanks(
-    msi_data::MSIData,
-    blank_spectrum_tag::Symbol,
-    snr_threshold::Float64;
-    frequency_threshold::Float64=0.5, # e.g., peak must be in 50% of blanks
-    bin_tolerance::Float64=0.005, # m/z tolerance for binning blank peaks
-    bin_tolerance_unit::Symbol=:mz
-    )::Vector{Float64}
-    if !(0 < frequency_threshold <= 1.0)
-        throw(ArgumentError("`frequency_threshold` must be between 0 and 1 (exclusive of 0). Got: $frequency_threshold"))
-    end
-    println("Identifying matrix peaks from blank spectra (tag: $blank_spectrum_tag)...")
-    blank_indices = Int[]
-    for (i, meta) in enumerate(msi_data.spectra_metadata)
-        if meta.type == blank_spectrum_tag
-            push!(blank_indices, i)
-        end
-    end
-
-    if isempty(blank_indices)
-        @warn "No blank spectra found with tag: $blank_spectrum_tag. Cannot identify matrix peaks."
-        return Float64[]
-    end
-
-    all_blank_peaks = Vector{NamedTuple}[]
-    num_blank_spectra = 0
-
-    # Collect peaks from each blank spectrum
-    _iterate_spectra_fast(msi_data, blank_indices) do idx, mz, intensity
-        num_blank_spectra += 1
-        if !validate_spectrum(mz, intensity)
-            @warn "Blank spectrum $idx is invalid, skipping peak detection for it."
-            push!(all_blank_peaks, NamedTuple[]) # Add empty list to maintain count
-            return
-        end
-        # Assuming profile mode for matrix peak detection
-        peaks = detect_peaks_profile(mz, intensity; snr_threshold=snr_threshold)
-        push!(all_blank_peaks, peaks)
-    end
-
-    if isempty(all_blank_peaks) || all(isempty, all_blank_peaks)
-        @warn "No peaks detected in any blank spectra. Cannot identify matrix peaks."
-        return Float64[]
-    end
-
-    # Flatten all peaks from blank spectra for initial binning
-    flat_blank_peaks = NamedTuple[]
-    for peaks_in_spec in all_blank_peaks
-        append!(flat_blank_peaks, peaks_in_spec)
-    end
-    sort!(flat_blank_peaks, by=p->p.mz)
-
-    # Bin the detected peaks from all blanks to find common m/z features
-    # This is a simplified binning for matrix peak identification
-    binned_matrix_features = Dict{Float64, Int}() # mz_center => count of spectra it appeared in
-
-    i = 1
-    while i <= length(flat_blank_peaks)
-        current_bin_start_idx = i
-        current_peak = flat_blank_peaks[i]
-        
-        # Calculate dynamic tolerance if in PPM
-        current_bin_mz = current_peak.mz
-        tol_val = (bin_tolerance_unit == :ppm) ? (current_bin_mz * bin_tolerance / 1e6) : bin_tolerance
-        
-        j = i + 1
-        while j <= length(flat_blank_peaks) && (flat_blank_peaks[j].mz - current_peak.mz) <= tol_val
-            j += 1
-        end
-        current_bin_end_idx = j - 1
-
-        # Calculate a representative m/z for the bin (e.g., intensity-weighted average)
-        peaks_in_bin = flat_blank_peaks[current_bin_start_idx:current_bin_end_idx]
-        if !isempty(peaks_in_bin)
-            sum_intensity = sum(p.intensity for p in peaks_in_bin)
-            if sum_intensity > 0
-                bin_center_mz = sum(p.mz * p.intensity for p in peaks_in_bin) / sum(sum_intensity)
-            else
-                bin_center_mz = mean(p.mz for p in peaks_in_bin)
-            end
-            
-            # Check how many blank spectra this feature appeared in
-            spectra_count = 0
-            # For each blank spectrum, check if it contains a peak within the current bin's tolerance
-            for spec_peaks in all_blank_peaks
-                if any(p -> abs(p.mz - bin_center_mz) <= tol_val, spec_peaks)
-                    spectra_count += 1
-                end
-            end
-            binned_matrix_features[bin_center_mz] = spectra_count
-        end
-        i = j
-    end
-
-    # Filter based on frequency_threshold
-    matrix_peak_mzs = Float64[]
-    min_spectra_count = ceil(Int, num_blank_spectra * frequency_threshold)
-
-    for (mz_center, count) in binned_matrix_features
-        if count >= min_spectra_count
-            push!(matrix_peak_mzs, mz_center)
-        end
-    end
-    sort!(matrix_peak_mzs)
-
-    println("Identified $(length(matrix_peak_mzs)) potential matrix peaks from $(num_blank_spectra) blank spectra.")
-    return matrix_peak_mzs
-end
-
-
 # =============================================================================
 # 4) Peak Detection
 # =============================================================================
 
 """
-    detect_peaks_profile(mz, y; ...) -> Vector{NamedTuple}
+    detect_peaks_profile_core(mz, y; ...) -> Vector{NamedTuple}
 
 Enhanced peak detection for profile-mode spectra with advanced filtering and quality metrics.
 
@@ -748,7 +699,7 @@ Returns a vector of NamedTuples, each representing a detected peak with:
 - `snr`: Signal-to-Noise Ratio
 - `prominence`: Peak prominence
 """
-function detect_peaks_profile(mz::AbstractVector{<:Real}, y::AbstractVector{<:Real}; 
+function detect_peaks_profile_core(mz::AbstractVector{<:Real}, y::AbstractVector{<:Real}; 
                               half_window::Int=10,
                               snr_threshold::Float64=2.0,
                               min_peak_prominence::Float64=0.1,
@@ -761,18 +712,32 @@ function detect_peaks_profile(mz::AbstractVector{<:Real}, y::AbstractVector{<:Re
     n = length(y)
     n < 3 && return NamedTuple{(:mz, :intensity, :fwhm, :shape_r2, :snr, :prominence), Tuple{Float64, Float64, Float64, Float64, Float64, Float64}}[]
     
-    noise_level = mad(y, normalize=true) + eps(Float64)
+    # Fast, non-allocating noise estimation
-    ys = smooth_spectrum(y; method=:savitzky_golay, window=max(5, 2*half_window+1), order=2) # Use smoothed data for detection
+    mean_y = sum(y) / n
+    noise_level = (sum(abs.(y .- mean_y)) / n) * 1.5 + eps(Float64)
+    
+    ys = smooth_spectrum_core(y; method=:savitzky_golay, window=max(5, 2*half_window+1), order=2)
 
     candidate_peak_indices = Int[]
-    for i in 2:n-1
+    sizehint!(candidate_peak_indices, div(n, 10)) # Pre-allocate memory capacity
+    
+    @inbounds for i in 2:n-1
         left = max(1, i - half_window)
         right = min(n, i + half_window)
         
-        # Prominence check
+        # Avoid @view allocation in tight loop by manually computing minimums and maximums
-        prominence = ys[i] - max(minimum(@view ys[left:i]), minimum(@view ys[i:right]))
+        min_left = ys[left]
+        for j in left:i; min_left = min(min_left, ys[j]); end
         
-        if ys[i] >= maximum(@view ys[left:right]) && 
+        min_right = ys[i]
+        for j in i:right; min_right = min(min_right, ys[j]); end
+        
+        prominence = ys[i] - max(min_left, min_right)
+        
+        max_local = ys[left]
+        for j in left:right; max_local = max(max_local, ys[j]); end
+        
+        if ys[i] >= max_local && 
            (ys[i] > snr_threshold * noise_level) &&
            (prominence > min_peak_prominence * ys[i])
             push!(candidate_peak_indices, i)
@@ -809,7 +774,14 @@ function detect_peaks_profile(mz::AbstractVector{<:Real}, y::AbstractVector{<:Re
         
         left = max(1, p_idx - half_window)
         right = min(n, p_idx + half_window)
-        prominence = ys[p_idx] - max(minimum(@view ys[left:p_idx]), minimum(@view ys[p_idx:right]))
+        
+        min_left = ys[left]
+        for j in left:p_idx; min_left = min(min_left, ys[j]); end
+        
+        min_right = ys[p_idx]
+        for j in p_idx:right; min_right = min(min_right, ys[j]); end
+        
+        prominence = ys[p_idx] - max(min_left, min_right)
 
         push!(detected_peaks, (mz=peak_mz, intensity=peak_int, fwhm=fwhm_ppm, shape_r2=shape_r2, snr=peak_snr, prominence=prominence))
     end
@@ -818,16 +790,30 @@ function detect_peaks_profile(mz::AbstractVector{<:Real}, y::AbstractVector{<:Re
 end
 
 """
-    detect_peaks_wavelet(mz, intensity; ...) -> Vector{NamedTuple}
+    detect_peaks_wavelet_core(mz, intensity; ...) -> Vector{NamedTuple}
 
-Detects peaks using Continuous Wavelet Transform (CWT).
+Detects peaks using Continuous Wavelet Transform (CWT). CWT is effective at
+identifying peaks at different scales (widths), making it robust for complex spectra.
 
-Returns a vector of NamedTuples, each representing a detected peak with:
+# Arguments
-- `mz`: m/z value of the peak
+- `mz::AbstractVector`: The m/z values of the spectrum.
-- `intensity`: Intensity of the peak
+- `intensity::AbstractVector`: The intensity values of the spectrum.
-- `snr`: Signal-to-Noise Ratio (simplified)
+- `scales`: A range of scales to use for the CWT. Corresponds to the widths of
+  the features to be detected.
+- `snr_threshold`: The minimum Signal-to-Noise Ratio for a CWT-detected local maximum
+  in the original spectrum to be considered a peak.
+- `half_window`: Used for calculating peak quality metrics like FWHM and shape R^2.
+
+# Returns
+A vector of `NamedTuple`s, each representing a detected peak with:
+- `mz`: m/z value of the peak.
+- `intensity`: Intensity of the peak from the original spectrum.
+- `fwhm`: Full Width at Half Maximum (in ppm).
+- `shape_r2`: Goodness-of-fit to a Gaussian shape.
+- `snr`: Signal-to-Noise Ratio.
+- `prominence`: Peak prominence (estimated as peak intensity for this method).
 """
-function detect_peaks_wavelet(mz::AbstractVector, intensity::AbstractVector; scales=1:10, snr_threshold=3.0, half_window=10)::Vector{NamedTuple{(:mz, :intensity, :fwhm, :shape_r2, :snr, :prominence), Tuple{Float64, Float64, Float64, Float64, Float64, Float64}}}
+function detect_peaks_wavelet_core(mz::AbstractVector, intensity::AbstractVector; scales=1:10, snr_threshold=3.0, half_window=10)::Vector{NamedTuple{(:mz, :intensity, :fwhm, :shape_r2, :snr, :prominence), Tuple{Float64, Float64, Float64, Float64, Float64, Float64}}}
     n = length(intensity)
     n < 10 && return NamedTuple{(:mz, :intensity, :fwhm, :shape_r2, :snr, :prominence), Tuple{Float64, Float64, Float64, Float64, Float64, Float64}}[]
     
@@ -876,7 +862,7 @@ function detect_peaks_wavelet(mz::AbstractVector, intensity::AbstractVector; sca
 end
 
 """
-    detect_peaks_centroid(mz, y; ...) -> Vector{NamedTuple}
+    detect_peaks_centroid_core(mz, y; ...) -> Vector{NamedTuple}
 
 Filters peaks in centroid-mode data based on intensity threshold.
 
@@ -884,7 +870,7 @@ Returns a vector of NamedTuples, each representing a detected peak with:
 - `mz`: m/z value of the peak
 - `intensity`: Intensity of the peak
 """
-function detect_peaks_centroid(mz::AbstractVector{<:Real}, y::AbstractVector{<:Real}; snr_threshold::Float64=0.0)
+function detect_peaks_centroid_core(mz::AbstractVector{<:Real}, y::AbstractVector{<:Real}; snr_threshold::Float64=0.0)
     noise_level = mad(y, normalize=true) + eps(Float64)
     detected_peaks = NamedTuple{(:mz, :intensity, :fwhm, :shape_r2, :snr, :prominence), Tuple{Float64, Float64, Float64, Float64, Float64, Float64}}[]
     
@@ -903,11 +889,11 @@ end
 # =============================================================================
 
 """
-    align_peaks_lowess(ref_mz, tgt_mz; ...)
+    align_peaks_lowess_core(ref_mz, tgt_mz; ...)
 
 Enhanced peak alignment with PPM tolerance and other constraints.
 """
-function align_peaks_lowess(ref_mz::Vector{<:Real}, tgt_mz::Vector{<:Real};
+function align_peaks_lowess_core(ref_mz::Vector{<:Real}, tgt_mz::Vector{<:Real};
                             method::Symbol=:linear, # :linear, :lowess, or :ransac
                             span::Float64=0.75, # Span for LOWESS
                             tolerance::Float64=0.002,
@@ -1030,15 +1016,15 @@ function align_peaks_lowess(ref_mz::Vector{<:Real}, tgt_mz::Vector{<:Real};
 end
 
 """
-    find_calibration_peaks(mz, intensity, reference_masses; ...)
+    find_calibration_peaks_core(mz, intensity, reference_masses; ...)
 
 Finds peaks that match a list of reference masses.
 """
-function find_calibration_peaks(mz::AbstractVector, intensity::AbstractVector, reference_masses::AbstractVector; ppm_tolerance=20.0)
+function find_calibration_peaks_core(mz::AbstractVector, intensity::AbstractVector, reference_masses::AbstractVector; ppm_tolerance=20.0)
     matched_peaks = Dict{Float64, Float64}()
     
-    # detect_peaks_profile returns Vector{NamedTuple}, so we need to extract mz values
+    # detect_peaks_profile_core returns Vector{NamedTuple}, so we need to extract mz values
-    detected_peaks_list = detect_peaks_profile(mz, intensity)
+    detected_peaks_list = detect_peaks_profile_core(mz, intensity)
     
     # Extract only the m/z values into a new vector for easier processing
     detected_mz_values = [p.mz for p in detected_peaks_list]
@@ -1057,16 +1043,16 @@ function find_calibration_peaks(mz::AbstractVector, intensity::AbstractVector, r
 end
 
 """
-    calibrate_spectra(spectra, internal_standards; ...)
+    calibrate_spectra_core(spectra, internal_standards; ...)
 
 Calibrates spectra using internal standards.
 """
-function calibrate_spectra(spectra::Vector, internal_standards::Vector; ppm_tolerance=20.0)
+function calibrate_spectra_core(spectra::Vector, internal_standards::Vector; ppm_tolerance=20.0)
     calibrated_spectra = similar(spectra)
     for (i, spec) in enumerate(spectra)
         mz, intensity = spec[1], spec[2]
 
-        matched_peaks = find_calibration_peaks(mz, intensity, internal_standards; ppm_tolerance=ppm_tolerance)
+        matched_peaks = find_calibration_peaks_core(mz, intensity, internal_standards; ppm_tolerance=ppm_tolerance)
         if length(matched_peaks) < 2
             @warn "Spectrum $i: Not enough calibration peaks found. Skipping."
             calibrated_spectra[i] = spec
@@ -1092,16 +1078,16 @@ end
 # =============================================================================
 
 """
-    bin_peaks(all_pk_mz::Vector{Vector{Float64}},
+    bin_peaks_core(all_pk_mz::Vector{Vector{Float64}},
               all_pk_int::Vector{Vector{Float64}},
-              params::PeakBinningParams) -> Tuple{FeatureMatrix, Vector{Tuple{Float64,Float64}}}
+              params::PeakBinning) -> Tuple{FeatureMatrix, Vector{Tuple{Float64,Float64}}}
 
 Enhanced peak binning with adaptive and PPM-based parameters, or uniform binning.
 
 # Arguments
 - `all_pk_mz`: A vector of m/z vectors for all spectra.
 - `all_pk_int`: A vector of intensity vectors for all spectra.
-- `params`: A `PeakBinningParams` struct.
+- `params`: A `PeakBinning` struct.
 
 # Returns
 - `Tuple{FeatureMatrix, Vector{Tuple{Float64,Float64}}}`: A tuple containing the generated FeatureMatrix and the bin definitions.
@@ -1111,8 +1097,8 @@ The use of `Threads.@threads` in the `:adaptive` and `:uniform` methods is safe.
 - In the `:adaptive` method, the loop is over the bins (`j` index). Each thread writes only to its assigned column `X[:, j]`, so there are no write conflicts between threads.
 - In the `:uniform` method, the loop is over the spectra (`s_idx`). Writes to `X[s_idx, bin_idx]` could theoretically conflict if different peaks from the same spectrum (`s_idx`) are processed by different threads. However, the loop is over `s_idx`, meaning each thread handles a distinct spectrum, making writes to `X[s_idx, :]` exclusive to that thread and thus safe.
 """
-function bin_peaks(spectra::Vector{MutableSpectrum},
+function bin_peaks_core(spectra::Vector{MutableSpectrum},
-                   params::PeakBinningParams)
+                   params::PeakBinning)
     ns = length(spectra) # Number of spectra
     ns == 0 && return FeatureMatrix(zeros(0,0), Tuple{Float64,Float64}[], Int[]), Tuple{Float64,Float64}[]
 

src/PreprocessingPipeline.jl

@@ -0,0 +1,510 @@
+# src/PreprocessingPipeline.jl
+
+using Base.Threads # For multithreading
+using Printf # For @sprintf
+using Interpolations # For linear_interpolation
+using DataFrames # For saving feature matrix
+using CSV # For saving feature matrix
+
+# This file provides a set of functions that apply preprocessing steps to a vector of
+# `MutableSpectrum` objects. Each function takes the vector of spectra and a dictionary
+# of parameters, modifying the spectra in-place where appropriate. This mirrors the
+# logic from `test/run_preprocessing.jl` but is intended for use in the main application.
+
+# ===================================================================
+# PREPROCESSING PIPELINE FUNCTIONS (IN-PLACE)
+# ===================================================================
+
+"""
+    apply_baseline_correction(spectra::Vector{MutableSpectrum}, params::Dict)
+
+Applies baseline correction to the intensity data of each spectrum. This function
+modifies the `.intensity` field of each `MutableSpectrum` object in-place.
+
+# Parameters from `params` Dict:
+- `:method` (Symbol): The algorithm to use. Supports `:snip`, `:convex_hull`, `:median`. Defaults to `:snip`.
+- `:iterations` (Int): The number of iterations for the SNIP algorithm. Defaults to 100.
+- `:window` (Int): The window size for the Median algorithm. Defaults to 20.
+"""
+function apply_baseline_correction(spectra::Vector{MutableSpectrum}, params::Dict)
+    method = get(params, :method, :snip)
+    iterations = get(params, :iterations, 100)
+    window = get(params, :window, 20)
+
+    Threads.@threads for s in spectra
+        if validate_spectrum(s.mz, s.intensity)
+            original_length = length(s.intensity)
+            baseline = apply_baseline_correction_core(s.intensity; method=method, iterations=iterations, window=window)
+            
+            # CRITICAL: Ensure baseline has the same length as intensity
+            if length(baseline) != original_length
+                @warn "Baseline correction: length mismatch for spectrum $(s.id). baseline=$(length(baseline)), intensity=$original_length. Skipping this spectrum."
+                continue
+            end
+            
+            s.intensity = max.(0.0, s.intensity .- baseline)
+        end
+    end
+end
+
+"""
+    apply_intensity_transformation(spectra::Vector{MutableSpectrum}, params::Dict)
+
+Applies an intensity transformation to the intensity data of each spectrum. This function
+modifies the `.intensity` field of each `MutableSpectrum` object in-place.
+
+# Parameters from `params` Dict:
+- `:method` (Symbol): The transformation to apply. Supports `:sqrt`, `:log`, `:log2`, `:log10`, `:log1p`. Defaults to `:sqrt`.
+"""
+function apply_intensity_transformation(spectra::Vector{MutableSpectrum}, params::Dict)
+    method = get(params, :method, :sqrt)
+
+    Threads.@threads for s in spectra
+        if validate_spectrum(s.mz, s.intensity)
+            original_length = length(s.intensity)
+            transformed = transform_intensity_core(s.intensity; method=method)
+            
+            # CRITICAL: Ensure transformation preserves array length
+            if length(transformed) != original_length
+                @warn "Intensity transformation: length mismatch for spectrum $(s.id). transformed=$(length(transformed)), original=$original_length. Skipping this spectrum."
+                continue
+            end
+            
+            s.intensity = transformed
+        end
+    end
+end
+
+"""
+    apply_smoothing(spectra::Vector{MutableSpectrum}, params::Dict)
+
+Applies a smoothing filter to the intensity data of each spectrum. This function
+modifies the `.intensity` field of each `MutableSpectrum` object in-place.
+
+# Parameters from `params` Dict:
+- `:method` (Symbol): The smoothing algorithm. Supports `:savitzky_golay`, `:moving_average`. Defaults to `:savitzky_golay`.
+- `:window` (Int): The size of the smoothing window. Defaults to 9.
+- `:order` (Int): The polynomial order for the Savitzky-Golay filter. Defaults to 2.
+"""
+function apply_smoothing(spectra::Vector{MutableSpectrum}, params::Dict)
+    method = get(params, :method, :savitzky_golay)
+    window = get(params, :window, 9)
+    order = get(params, :order, 2)
+
+    Threads.@threads for s in spectra
+        if validate_spectrum(s.mz, s.intensity)
+            original_length = length(s.intensity)
+            smoothed_intensity = smooth_spectrum_core(s.intensity; method=method, window=window, order=order)
+            
+            # CRITICAL: Ensure smoothing preserves array length  
+            if length(smoothed_intensity) != original_length
+                @warn "Smoothing: length mismatch for spectrum $(s.id). smoothed=$(length(smoothed_intensity)), original=$original_length, method=$method, window=$window. Skipping this spectrum."
+                continue
+            end
+            
+            s.intensity = max.(0.0, smoothed_intensity)
+        end
+    end
+end
+
+"""
+    apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
+
+Detects peaks in each spectrum and stores them in the `.peaks` field of each
+`MutableSpectrum` object, modifying it in-place.
+
+# Parameters from `params` Dict:
+- `:method` (Symbol): The peak detection algorithm. Supports `:profile`, `:wavelet`, `:centroid`. Defaults to `:profile`.
+- `:snr_threshold` (Float64): Signal-to-Noise Ratio threshold.
+- `:half_window` (Int): Half-window size for local maxima detection.
+- `:min_peak_prominence` (Float64): Minimum required prominence for a peak.
+- `:merge_peaks_tolerance` (Float64): m/z tolerance to merge adjacent peaks.
+"""
+function apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
+    method = get(params, :method, :profile)
+    snr_threshold = get(params, :snr_threshold, 3.0)
+    half_window = get(params, :half_window, 10)
+    min_peak_prominence = get(params, :min_peak_prominence, 0.1)
+    merge_peaks_tolerance = get(params, :merge_peaks_tolerance, 0.002)
+
+    Threads.@threads for s in spectra
+        if validate_spectrum(s.mz, s.intensity)
+            # old_len = length(s.peaks)
+            if method == :profile
+                s.peaks = detect_peaks_profile_core(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window, min_peak_prominence=min_peak_prominence, merge_peaks_tolerance=merge_peaks_tolerance)
+            elseif method == :wavelet
+                s.peaks = detect_peaks_wavelet_core(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
+            elseif method == :centroid
+                s.peaks = detect_peaks_centroid_core(s.mz, s.intensity; snr_threshold=snr_threshold)
+            else
+                s.peaks = detect_peaks_profile_core(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
+            end
+            # @info "Spectrum $(s.id): detected $(length(s.peaks)) peaks"
+        else
+            s.peaks = []
+        end
+    end
+end
+
+"""
+    apply_peak_selection(spectra::Vector{MutableSpectrum}, params::Dict)
+
+Filters peaks within each spectrum based on quality criteria. This step removes peaks
+that do not meet the specified thresholds for signal-to-noise ratio (SNR),
+full width at half maximum (FWHM), and peak shape.
+
+This function modifies the `.peaks` field of each `MutableSpectrum` object in the `spectra` vector in-place.
+
+# Parameters from `params` Dict:
+- `:min_snr` (Float64): The minimum Signal-to-Noise Ratio required for a peak to be kept.
+- `:min_fwhm_ppm` (Float64): The minimum FWHM (in ppm) for a peak.
+- `:max_fwhm_ppm` (Float64): The maximum FWHM (in ppm) for a peak.
+- `:min_shape_r2` (Float64): The minimum R² value from a Gaussian fit, measuring peak shape quality.
+"""
+function apply_peak_selection(spectra::Vector{MutableSpectrum}, params::Dict)
+    min_snr = get(params, :min_snr, 0.0)
+    min_fwhm = get(params, :min_fwhm_ppm, 0.0)
+    max_fwhm = get(params, :max_fwhm_ppm, Inf)
+    min_r2 = get(params, :min_shape_r2, 0.0)
+
+    # Handle `nothing` from params, which can happen if pre-calculation fails.
+    min_snr = isnothing(min_snr) ? 0.0 : min_snr
+    min_fwhm = isnothing(min_fwhm) ? 0.0 : min_fwhm
+    max_fwhm = isnothing(max_fwhm) ? Inf : max_fwhm
+    min_r2 = isnothing(min_r2) ? 0.0 : min_r2
+
+    Threads.@threads for s in spectra
+        if !isempty(s.peaks)
+            filter!(p -> 
+                p.snr >= min_snr &&
+                (min_fwhm <= p.fwhm <= max_fwhm) &&
+                p.shape_r2 >= min_r2,
+                s.peaks
+            )
+        end
+    end
+end
+
+"""
+    apply_calibration(spectra::Vector{MutableSpectrum}, params::Dict, reference_peaks::Dict)
+
+Performs mass calibration on each spectrum using a list of internal standards.
+This function modifies the `.mz` axis of each `MutableSpectrum` object in-place.
+
+# Parameters from `params` Dict:
+- `:method` (Symbol): The calibration method. Only `:internal_standards` is currently meaningful.
+- `:ppm_tolerance` (Float64): The tolerance in PPM for matching detected peaks to reference masses.
+- `:fit_order` (Int): The polynomial order for the calibration fit (not yet used in this implementation, defaults to linear).
+"""
+function apply_calibration(spectra::Vector{MutableSpectrum}, params::Dict, reference_peaks::Dict)
+    method = get(params, :method, :none)
+    ppm_tolerance = get(params, :ppm_tolerance, 20.0)
+
+    if method == :none || isempty(reference_peaks)
+        return
+    end
+
+    reference_masses = collect(keys(reference_peaks))
+
+    Threads.@threads for i in 1:length(spectra)
+        s = spectra[i]
+        if validate_spectrum(s.mz, s.intensity)
+            original_length = length(s.mz)
+            matched_peaks = find_calibration_peaks_core(s.mz, s.intensity, reference_masses; ppm_tolerance=ppm_tolerance)
+            if length(matched_peaks) >= 2
+                measured = sort(collect(values(matched_peaks)))
+                theoretical = sort(collect(keys(matched_peaks)))
+                itp = linear_interpolation(measured, theoretical, extrapolation_bc=Line())
+                new_mz = itp(s.mz)
+                
+                # CRITICAL: Ensure m/z axis preserves array length
+                if length(new_mz) != original_length
+                    @warn "Calibration: length mismatch for spectrum $(s.id). new_mz=$(length(new_mz)), original=$original_length. Skipping this spectrum."
+                    continue
+                end
+                
+                s.mz = new_mz # Modify mz-axis in-place
+            else
+                @warn "Spectrum $(s.id): insufficient reference peaks ($(length(matched_peaks)) found), skipping calibration."
+            end
+        end
+    end
+end
+
+"""
+    apply_peak_alignment(spectra::Vector{MutableSpectrum}, params::Dict)
+
+Aligns the m/z axis of all spectra to a chosen reference spectrum. This function
+modifies both the `.mz` axis and the m/z values within the `.peaks` field of each
+`MutableSpectrum` object in-place.
+
+# Parameters from `params` Dict:
+- `:method` (Symbol): The alignment algorithm. Supports `:lowess`, `:linear`, `:ransac`.
+- `:tolerance` (Float64): The tolerance for matching peaks between spectra.
+- `:tolerance_unit` (Symbol): The unit for tolerance, `:mz` or `:ppm`.
+"""
+function apply_peak_alignment(spectra::Vector{MutableSpectrum}, params::Dict)
+    method = get(params, :method, :none)
+    tolerance = get(params, :tolerance, 0.002)
+    tolerance_unit = get(params, :tolerance_unit, :mz)
+
+    if method == :none
+        return
+    end
+
+    ref_find_idx = findfirst(s -> !isempty(s.peaks), spectra)
+    if ref_find_idx === nothing
+        @warn "Insufficient spectra with peaks for alignment. Skipping."
+        return
+    end
+
+    ref_spectrum = spectra[ref_find_idx]
+    ref_peaks_mz = [p.mz for p in ref_spectrum.peaks]
+
+    Threads.@threads for s in spectra
+        if s.id == ref_spectrum.id || isempty(s.peaks)
+            continue
+        end
+        
+        current_peaks_mz = [p.mz for p in s.peaks]
+        alignment_func = align_peaks_lowess_core(ref_peaks_mz, current_peaks_mz; method=method, tolerance=tolerance, tolerance_unit=tolerance_unit)
+        
+        original_length = length(s.mz)
+        new_mz = alignment_func.(s.mz)
+        
+        # CRITICAL: Ensure alignment preserves array length
+        if length(new_mz) != original_length
+            @warn "Peak alignment: m/z length mismatch for spectrum $(s.id). new_mz=$(length(new_mz)), original=$original_length. Skipping this spectrum."
+            continue
+        end
+        
+        s.mz = new_mz # Update m/z axis
+        
+        # Update peak m/z values
+        for i in 1:length(s.peaks)
+            old_peak = s.peaks[i]
+            aligned_peak_mz = alignment_func(old_peak.mz)
+            s.peaks[i] = (mz=aligned_peak_mz, intensity=old_peak.intensity, fwhm=old_peak.fwhm, shape_r2=old_peak.shape_r2, snr=old_peak.snr, prominence=old_peak.prominence)
+        end
+    end
+end
+
+"""
+    apply_normalization(spectra::Vector{MutableSpectrum}, params::Dict)
+
+Applies intensity normalization to each spectrum. This function modifies the
+`.intensity` field of each `MutableSpectrum` object in-place.
+
+# Parameters from `params` Dict:
+- `:method` (Symbol): The normalization method. Supports `:tic`, `:median`, `:rms`, `:none`.
+"""
+function apply_normalization(spectra::Vector{MutableSpectrum}, params::Dict)
+    method = get(params, :method, :tic)
+
+    Threads.@threads for s in spectra
+        if validate_spectrum(s.mz, s.intensity)
+            original_length = length(s.intensity)
+            normalized = apply_normalization_core(s.intensity; method=method)
+            
+            # CRITICAL: Ensure normalization preserves array length
+            if length(normalized) != original_length
+                @warn "Normalization: length mismatch for spectrum $(s.id). normalized=$(length(normalized)), original=$original_length. Skipping this spectrum."
+                continue
+            end
+            
+            s.intensity = normalized
+        end
+    end
+end
+function apply_peak_binning(spectra::Vector{MutableSpectrum}, params::Dict)
+    tolerance = get(params, :tolerance, 20.0)
+    tolerance_unit = get(params, :tolerance_unit, :ppm)
+    min_peak_per_bin = get(params, :min_peak_per_bin, 3)
+
+    if isempty(spectra) || all(s -> isempty(s.peaks), spectra)
+        @warn "No peaks found for binning. Returning empty feature matrix."
+        return nothing, nothing
+    end
+        
+    all_peaks = Vector{Tuple{Float64, Float64}}()
+    for s in spectra
+        for p in s.peaks
+            push!(all_peaks, (p.mz, p.intensity))
+        end
+    end
+    
+    if isempty(all_peaks)
+        @warn "No peaks collected for binning."
+        return nothing, nothing
+    end
+    
+    sort!(all_peaks, by=x->x[1])
+    
+    bin_centers = Float64[]
+    bin_intensities = Float64[]
+    
+    i = 1
+    while i <= length(all_peaks)
+        current_bin_start = i
+        current_peak = all_peaks[i]
+        
+        j = i + 1
+        while j <= length(all_peaks)
+            next_peak = all_peaks[j]
+            tol = (tolerance_unit == :ppm) ? (current_peak[1] * tolerance / 1e6) : tolerance
+            
+            if (next_peak[1] - current_peak[1]) <= tol
+                j += 1
+            else
+                break
+            end
+        end
+        
+        current_bin_end = j - 1
+        bin_size = current_bin_end - current_bin_start + 1
+        
+        if bin_size >= min_peak_per_bin
+            bin_peaks = all_peaks[current_bin_start:current_bin_end]
+            
+            mz_sum = sum(p[1] for p in bin_peaks)
+            intensity_sum = sum(p[2] for p in bin_peaks)
+            
+            mz_center = mz_sum / bin_size
+            avg_intensity = intensity_sum / bin_size
+            
+            push!(bin_centers, mz_center)
+            push!(bin_intensities, avg_intensity)
+        end
+        
+        i = j
+    end
+    
+    if !isempty(bin_centers)
+        n_bins = length(bin_centers)
+        feature_matrix = Matrix{Float64}(undef, 2, n_bins)
+        
+        for i in 1:n_bins
+            feature_matrix[1, i] = bin_centers[i]
+            feature_matrix[2, i] = bin_intensities[i]
+        end
+        
+        bin_info = [(bin_centers[i], bin_intensities[i]) for i in 1:n_bins]
+        return feature_matrix, bin_info
+    else
+        @warn "No bins created after filtering"
+        return nothing, nothing
+    end
+end
+
+"""
+    save_feature_matrix(feature_matrix::Matrix{Float64}, bin_info, output_dir::String) -> Tuple{String, String}
+
+Saves the aggregated `2 x n_bins` feature matrix into two different CSV formats.
+
+1.  **Simple Format (`feature_matrix_simple.csv`):** A two-column CSV with "mz" and "intensity".
+2.  **Standard Format (`feature_matrix_standard.csv`):** A row-based format where m/z values are headers and there is a single data row for the aggregated spectrum.
+
+# Arguments
+- `feature_matrix::Matrix{Float64}`: The `2 x n_bins` matrix from `apply_peak_binning`.
+- `bin_info`: The associated bin information (currently unused but kept for compatibility).
+- `output_dir::String`: The directory where the output CSV files will be saved.
+
+# Returns
+- A tuple containing the paths to the two saved files.
+"""
+function save_feature_matrix(feature_matrix::Matrix{Float64}, bin_info, output_dir::String)
+    # Save as simple CSV with m/z and intensity rows
+    csv_path = joinpath(output_dir, "feature_matrix_simple.csv")
+    
+    open(csv_path, "w") do io
+        write(io, "mz,intensity\n")
+        for i in 1:size(feature_matrix, 2)
+            mz = feature_matrix[1, i]
+            intensity = feature_matrix[2, i]
+            write(io, "$mz,$intensity\n")
+        end
+    end
+    @info "Saved simple feature matrix: $csv_path"
+    
+    # Also save in a more standard format for MSI
+    csv_path_standard = joinpath(output_dir, "feature_matrix_standard.csv")
+    
+    open(csv_path_standard, "w") do io
+        write(io, "sample_type,")
+        mz_headers = [@sprintf("mz_%.4f", feature_matrix[1, i]) for i in 1:size(feature_matrix, 2)]
+        write(io, join(mz_headers, ",") * "\n")
+        
+        write(io, "aggregated_spectrum,")
+        intensity_values = [feature_matrix[2, i] for i in 1:size(feature_matrix, 2)]
+        write(io, join(string.(intensity_values), ",") * "\n")
+    end
+    @info "Saved standard format matrix: $csv_path_standard"
+    
+    return csv_path, csv_path_standard
+end
+
+function execute_full_preprocessing(progress_callback::Function, # Now first positional
+                                   spectra::Vector{MutableSpectrum}, params::Dict, 
+                                   pipeline_steps::Vector{String}, reference_peaks::Dict, 
+                                   mask_path::Union{String, Nothing} # Positional argument
+                                   )
+    
+    println("Starting preprocessing pipeline with $(length(spectra)) spectra")
+    println("Steps: $(join(pipeline_steps, " -> "))")
+    
+    # These variables will be populated by the pipeline steps
+    feature_matrix = nothing
+    bin_definitions = nothing
+
+    # Apply pipeline steps, modifying `spectra` in-place
+    for step in pipeline_steps
+        progress_callback(step)
+        println("\n" * "-"^60)
+        println("PROCESSING STEP: $step")
+        println("-"^60)
+        
+        if step == "stabilization"
+            println("  Applying intensity transformation (stabilization)")
+            apply_intensity_transformation(spectra, get(params, :Stabilization, Dict()))
+            
+        elseif step == "baseline_correction"
+            println("  Applying baseline correction")
+            apply_baseline_correction(spectra, get(params, :BaselineCorrection, Dict()))
+            
+        elseif step == "smoothing"
+            println("  Applying smoothing")
+            apply_smoothing(spectra, get(params, :Smoothing, Dict()))
+            
+        elseif step == "peak_picking"
+            println("  Applying peak picking")
+            apply_peak_picking(spectra, get(params, :PeakPicking, Dict()))
+
+        elseif step == "peak_selection"
+            println("  Applying peak selection")
+            apply_peak_selection(spectra, get(params, :PeakSelection, Dict()))
+            
+        elseif step == "calibration"
+            println("  Applying calibration")
+            apply_calibration(spectra, get(params, :Calibration, Dict()), reference_peaks)
+            
+        elseif step == "peak_alignment"
+            println("  Applying peak alignment")
+            apply_peak_alignment(spectra, get(params, :PeakAlignment, Dict()))
+            
+        elseif step == "normalization"
+            println("  Applying normalization")
+            apply_normalization(spectra, get(params, :Normalization, Dict()))
+            
+        elseif step == "peak_binning"
+            println("  Applying peak binning")
+            feature_matrix, bin_definitions = apply_peak_binning(spectra, get(params, :PeakBinning, Dict()))
+            
+        else
+            @warn "Unknown step: $step, skipping"
+        end
+        
+        println("✓ Completed step: $step")
+    end
+    
+    return feature_matrix, bin_definitions
+end

src/ResourcePool.jl

@@ -0,0 +1,79 @@
+# src/ResourcePool.jl
+using Base.Threads
+
+"""
+    ResourcePool{T}
+
+A thread-safe pool for reusing objects of type `T` to minimize allocations and GC pressure.
+Specifically designed for high-performance computing tasks where large buffers are needed
+repeatedly across multiple threads.
+
+# Fields:
+- `pool::Vector{T}`: The underlying storage for idle resources.
+- `lock::ReentrantLock`: Ensures thread-safe access to the pool.
+- `max_size::Int`: Maximum number of resources to hold in the pool.
+- `constructor::Function`: A function to create a new resource if the pool is empty.
+"""
+mutable struct ResourcePool{T}
+    pool::Vector{T}
+    lock::ReentrantLock
+    max_size::Int
+    constructor::Function
+end
+
+
+
+"""
+    ResourcePool{T}(constructor::Function; max_size::Int=2 * nthreads())
+
+Creates a new `ResourcePool` for resources of type `T`.
+"""
+function ResourcePool{T}(constructor::Function; max_size::Int=2 * nthreads()) where T
+    return ResourcePool{T}(T[], ReentrantLock(), max_size, constructor)
+end
+
+"""
+    acquire(pool::ResourcePool{T}) -> T
+
+Retrieves a resource from the pool. If the pool is empty, a new resource is created
+using the constructor.
+"""
+function acquire(pool::ResourcePool{T}) where T
+    lock(pool.lock) do
+        if !isempty(pool.pool)
+            return pop!(pool.pool)
+        end
+    end
+    # Create new resource outside of lock to minimize contention
+    return pool.constructor()
+end
+
+"""
+    release!(pool::ResourcePool{T}, resource::T)
+
+Returns a resource to the pool for later reuse. If the pool is already at `max_size`,
+the resource is allowed to be garbage collected.
+"""
+function release!(pool::ResourcePool{T}, resource::T) where T
+    lock(pool.lock) do
+        if length(pool.pool) < pool.max_size
+            push!(pool.pool, resource)
+        end
+    end
+    return nothing
+end
+
+"""
+    with_resource(f::Function, pool::ResourcePool{T})
+
+Acquires a resource from the pool, executes the function `f(resource)`, and
+automatically releases the resource back to the pool when finished.
+"""
+function with_resource(f::Function, pool::ResourcePool{T}) where T
+    resource = acquire(pool)
+    try
+        return f(resource)
+    finally
+        release!(pool, resource)
+    end
+end

src/StreamingKernels.jl

@@ -0,0 +1,317 @@
+# src/StreamingKernels.jl
+# ============================================================================
+# In-Place Spectral Kernels for the Streaming Pipeline
+# 
+# These functions operate on raw (mz, intensity) views from the Sprint 1
+# Mmap engine. They write results back to the input buffers using .= to
+# achieve zero-allocation processing per spectrum.
+#
+# Design contract:
+#   - All !-suffixed functions modify their arguments in-place
+#   - If a kernel needs temporary storage, it borrows from data.resource_pool
+#   - No function creates MutableSpectrum objects
+# ============================================================================
+
+using Statistics: mean, median
+
+# =============================================================================
+# Category A: Purely Streamable Kernels
+# =============================================================================
+
+"""
+    normalize_inplace!(intensity::AbstractVector{<:Real}, method::Symbol)
+
+Normalizes intensity values in-place. Supports :tic, :median, :rms.
+Zero-allocation for the normalization itself.
+"""
+@inline function normalize_inplace!(intensity::AbstractVector{<:Real}, method::Symbol)
+    if method === :tic
+        s = sum(intensity)
+        if s > 0
+            intensity ./= s
+        end
+    elseif method === :median
+        m = median(intensity)
+        if m > 0
+            intensity ./= m
+        end
+    elseif method === :rms
+        s = sqrt(sum(abs2, intensity) / length(intensity))
+        if s > 0
+            intensity ./= s
+        end
+    end
+    return intensity
+end
+
+"""
+    transform_inplace!(intensity::AbstractVector{Float64}, method::Symbol)
+
+Applies intensity transformation in-place. Supports :sqrt, :log1p, :log, :log2, :log10.
+"""
+@inline function transform_inplace!(intensity::AbstractVector{Float64}, method::Symbol)
+    if method === :sqrt
+        @inbounds @simd for i in eachindex(intensity)
+            intensity[i] = sqrt(max(0.0, intensity[i]))
+        end
+    elseif method === :log1p
+        @inbounds @simd for i in eachindex(intensity)
+            intensity[i] = log1p(max(0.0, intensity[i]))
+        end
+    elseif method === :log
+        @inbounds @simd for i in eachindex(intensity)
+            intensity[i] = log(max(eps(Float64), intensity[i]))
+        end
+    elseif method === :log2
+        @inbounds @simd for i in eachindex(intensity)
+            intensity[i] = log2(max(eps(Float64), intensity[i]))
+        end
+    elseif method === :log10
+        @inbounds @simd for i in eachindex(intensity)
+            intensity[i] = log10(max(eps(Float64), intensity[i]))
+        end
+    end
+    return intensity
+end
+
+"""
+    smooth_inplace!(intensity::AbstractVector{Float64}, data::MSIData;
+                    method::Symbol=:savitzky_golay, window::Int=9, order::Int=2)
+
+Smooths intensity in-place using a temporary buffer from the resource pool.
+The SavitzkyGolay library allocates internally, but we copy the result back
+to the original buffer and return the pool buffer.
+"""
+function smooth_inplace!(intensity::AbstractVector{Float64}, scratch::AbstractVector{Float64}, data::MSIData;
+                         method::Symbol=:savitzky_golay, window::Int=9, order::Int=2)
+    n = length(intensity)
+    if n < 3
+        return intensity
+    end
+
+    if method === :savitzky_golay
+        win = isodd(window) ? window : window + 1
+        if n < win
+            return intensity
+        end
+        # SavitzkyGolay handles its own math but causes mild allocation.
+        res = SavitzkyGolay.savitzky_golay(collect(intensity), win, order)
+        @inbounds for i in eachindex(intensity)
+            intensity[i] = max(0.0, res.y[i])
+        end
+    elseif method === :moving_average
+        copyto!(scratch, intensity)
+        
+        half_w = div(window, 2)
+        @inbounds for i in 1:n
+            s_idx = max(1, i - half_w)
+            e_idx = min(n, i + half_w)
+            s = 0.0
+            @simd for j in s_idx:e_idx
+                s += scratch[j]
+            end
+            intensity[i] = max(0.0, s / (e_idx - s_idx + 1))
+        end
+    end
+    return intensity
+end
+
+"""
+    baseline_subtract_inplace!(intensity::AbstractVector{Float64}, data::MSIData;
+                               method::Symbol=:snip, iterations::Int=100, window::Int=20)
+
+Subtracts baseline from intensity in-place. Uses two pool buffers for the
+SNIP ping-pong iteration to avoid any heap allocation in the hot loop.
+"""
+function baseline_subtract_inplace!(intensity::AbstractVector{Float64}, scratch::AbstractVector{Float64}, data::MSIData;
+                                    method::Symbol=:snip, iterations::Int=100, window::Int=20)
+    n = length(intensity)
+    if n < 3
+        return intensity
+    end
+    
+    if method === :snip
+        copyto!(scratch, intensity)
+        
+        for k in 1:iterations
+            prev_val = scratch[1]
+            scratch[1] = min(scratch[1], scratch[2])
+            
+            @inbounds for i in 2:n-1
+                curr_val = scratch[i]
+                scratch[i] = min(curr_val, 0.5 * (prev_val + scratch[i+1]))
+                prev_val = curr_val
+            end
+            scratch[n] = min(scratch[n], prev_val)
+        end
+        
+        @inbounds @simd for i in 1:n
+            intensity[i] = max(0.0, intensity[i] - scratch[i])
+        end
+    elseif method === :convex_hull
+        baseline = convex_hull_baseline(intensity)
+        @inbounds @simd for i in eachindex(intensity)
+            intensity[i] = max(0.0, intensity[i] - baseline[i])
+        end
+    elseif method === :median
+        baseline = median_baseline(intensity; window=window)
+        @inbounds @simd for i in eachindex(intensity)
+            intensity[i] = max(0.0, intensity[i] - baseline[i])
+        end
+    end
+    return intensity
+end
+
+"""
+    detect_peaks_streaming(mz::AbstractVector, intensity::AbstractVector;
+                          method::Symbol=:profile, snr_threshold::Float64=3.0,
+                          half_window::Int=10, min_peak_prominence::Float64=0.1,
+                          merge_peaks_tolerance::Float64=0.002)
+
+Detects peaks and returns a vector of (mz, intensity) tuples.
+This delegates to existing _core functions but returns a lightweight format
+suitable for sparse accumulation (no NamedTuple overhead in the hot path).
+"""
+function detect_peaks_streaming(callback::Function, mz::AbstractVector{Float64}, intensity::AbstractVector{Float64}, scratch::AbstractVector{Float64};
+                               method::Symbol=:profile, snr_threshold::Float64=3.0,
+                               half_window::Int=10, min_peak_prominence::Float64=0.1,
+                               merge_peaks_tolerance::Float64=0.002)
+    n = length(intensity)
+    if n < 3
+        return
+    end
+
+    if method === :profile || method === :wavelet
+        # Zero-allocation noisy estimation (using mean of bottom half)
+        sum_i = 0.0
+        @simd for i in 1:n
+            sum_i += intensity[i]
+        end
+        mean_i = sum_i / n
+        
+        sum_noise = 0.0
+        count_noise = 0
+        @inbounds for i in 1:n
+            if intensity[i] < mean_i
+                sum_noise += intensity[i]
+                count_noise += 1
+            end
+        end
+        # Use * 1.5 as an approximation to MAD
+        noise_level = count_noise > 0 ? (sum_noise / count_noise) * 1.5 + eps(Float64) : mean_i + eps(Float64)
+        
+        # We will use the scratch buffer to store candidate indices to avoid allocating `Int[]`
+        # Because scratch is Float64, we can safely store integer indices up to 2^53 exactly.
+        num_candidates = 0
+        
+        @inbounds for i in 2:n-1
+            if intensity[i] > snr_threshold * noise_level
+                left = max(1, i - half_window)
+                right = min(n, i + half_window)
+                
+                is_max = true
+                for j in left:right
+                    if intensity[j] > intensity[i]
+                        is_max = false
+                        break
+                    end
+                end
+                
+                if is_max
+                    # Compute prominence
+                    min_left = intensity[i]
+                    for j in left:i
+                        if intensity[j] < min_left
+                            min_left = intensity[j]
+                        end
+                    end
+                    min_right = intensity[i]
+                    for j in i:right
+                        if intensity[j] < min_right
+                            min_right = intensity[j]
+                        end
+                    end
+                    
+                    prominence = intensity[i] - max(min_left, min_right)
+                    
+                    if prominence > min_peak_prominence * intensity[i]
+                        num_candidates += 1
+                        scratch[num_candidates] = i
+                    end
+                end
+            end
+        end
+        
+        # Merge close peaks
+        if num_candidates > 0
+            if merge_peaks_tolerance > 0
+                last_idx = trunc(Int, scratch[1])
+                # We emit the first peak lazily down below, so let's compact them in place
+                num_merged = 1
+                
+                for i in 2:num_candidates
+                    idx = trunc(Int, scratch[i])
+                    if (mz[idx] - mz[last_idx]) > merge_peaks_tolerance
+                        num_merged += 1
+                        scratch[num_merged] = idx
+                        last_idx = idx
+                    elseif intensity[idx] > intensity[last_idx]
+                        scratch[num_merged] = idx
+                        last_idx = idx
+                    end
+                end
+                num_candidates = num_merged
+            end
+            
+            # Emit merged peaks
+            for i in 1:num_candidates
+                idx = trunc(Int, scratch[i])
+                callback(mz[idx], intensity[idx])
+            end
+        end
+        
+    elseif method === :centroid
+        # Just use any value over snr_threshold * mean_noise
+        sum_i = sum(intensity)
+        mean_i = sum_i / n
+        noise_level = mean_i + eps(Float64)
+        
+        @inbounds for i in 1:n
+            if intensity[i] > snr_threshold * noise_level
+                callback(mz[i], intensity[i])
+            end
+        end
+    end
+end
+
+# =============================================================================
+# Category B: Conditionally Streamable Kernels (Fixed-Reference)
+# =============================================================================
+
+"""
+    calibrate_inplace!(mz::Vector{Float64}, intensity::AbstractVector,
+                       reference_masses::Vector{Float64}; ppm_tolerance::Float64=20.0)
+
+Calibrates the m/z axis in-place using a fixed dictionary of internal standard
+reference masses. This is streamable because the reference is constant.
+
+Returns `true` if calibration was applied, `false` if insufficient peaks were found.
+"""
+function calibrate_inplace!(mz::Vector{Float64}, intensity::AbstractVector,
+                            reference_masses::Vector{Float64}; ppm_tolerance::Float64=20.0)
+    matched_peaks = find_calibration_peaks_core(mz, intensity, reference_masses; 
+                                                 ppm_tolerance=ppm_tolerance)
+    if length(matched_peaks) < 2
+        return false # Insufficient reference peaks
+    end
+    
+    measured = sort(collect(values(matched_peaks)))
+    theoretical = sort(collect(keys(matched_peaks)))
+    itp = linear_interpolation(measured, theoretical, extrapolation_bc=Line())
+    
+    # Apply calibration in-place
+    @inbounds for i in eachindex(mz)
+        mz[i] = itp(mz[i])
+    end
+    return true
+end

src/StreamingPipeline.jl

@@ -0,0 +1,402 @@
+# src/StreamingPipeline.jl
+# ============================================================================
+# The Streaming Pipeline Executor
+#
+# This module provides `process_dataset!`, the Sprint 2 master function that
+# streams spectral data through an in-place kernel chain and accumulates
+# results into a SparseMatrixCSC without ever holding more than 1 spectrum
+# per thread in RAM.
+#
+# Architecture:
+#   1. _iterate_spectra_fast → Mmap zero-copy views
+#   2. copyto!(writable_buf, view) → makes mutable copy for kernels
+#   3. Kernel chain: smooth! → baseline! → peaks → bin
+#   4. Thread-local (I, J, V) sparse accumulators
+#   5. Final sparse(I, J, V, num_bins, num_spectra) assembly
+#
+# This works alongside the existing execute_full_preprocessing in
+# PreprocessingPipeline.jl — it does NOT replace the app.jl integration.
+# ============================================================================
+
+using SparseArrays
+using Printf
+
+# =============================================================================
+# Configuration Structs
+# =============================================================================
+
+"""
+    StreamingStep
+
+Represents a single step in the streaming pipeline.
+"""
+struct StreamingStep
+    name::Symbol
+    params::Dict{Symbol, Any}
+end
+
+"""
+    PipelineConfig
+
+Holds the complete configuration for a streaming pipeline execution.
+
+# Fields
+- `steps::Vector{StreamingStep}` — ordered sequence of processing steps
+- `reference_peaks::Vector{Float64}` — fixed m/z values for calibration (Category B)
+- `num_bins::Int` — number of bins for the output feature matrix
+- `min_peaks_per_bin::Int` — minimum peak count to keep a bin
+- `frequency_threshold::Float64` — minimum fraction of spectra a bin must appear in (0.0-1.0)
+
+# Example
+```julia
+config = PipelineConfig(
+    steps = [
+        StreamingStep(:smoothing, Dict(:method => :savitzky_golay, :window => 9, :order => 2)),
+        StreamingStep(:baseline_correction, Dict(:method => :snip, :iterations => 100)),
+        StreamingStep(:normalization, Dict(:method => :tic)),
+        StreamingStep(:peak_picking, Dict(:method => :profile, :snr_threshold => 3.0)),
+    ],
+    num_bins = 2000
+)
+```
+"""
+struct PipelineConfig
+    steps::Vector{StreamingStep}
+    reference_peaks::Vector{Float64}
+    num_bins::Int
+    min_peaks_per_bin::Int
+    frequency_threshold::Float64
+end
+
+# Convenience constructor with defaults
+function PipelineConfig(; steps::Vector{StreamingStep}=StreamingStep[],
+                         reference_peaks::Vector{Float64}=Float64[],
+                         num_bins::Int=2000,
+                         min_peaks_per_bin::Int=3,
+                         frequency_threshold::Float64=0.0)
+    return PipelineConfig(steps, reference_peaks, num_bins, min_peaks_per_bin, frequency_threshold)
+end
+
+# =============================================================================
+# Sparse Accumulator (Thread-Local)
+# =============================================================================
+
+"""
+    SparseAccumulator
+
+Thread-local accumulator for sparse matrix construction.
+Collects (row, col, val) triplets that will be assembled into
+a SparseMatrixCSC at the end of the pipeline.
+"""
+mutable struct SparseAccumulator
+    I::Vector{Int}    # Row indices (bin indices)
+    J::Vector{Int}    # Column indices (spectrum indices)
+    V::Vector{Float64} # Values (intensities)
+    lck::Base.Threads.SpinLock
+
+    function SparseAccumulator(capacity_hint::Int=10000)
+        acc = new(
+            Vector{Int}(undef, 0),
+            Vector{Int}(undef, 0),
+            Vector{Float64}(undef, 0),
+            Base.Threads.SpinLock()
+        )
+        sizehint!(acc.I, capacity_hint)
+        sizehint!(acc.J, capacity_hint)
+        sizehint!(acc.V, capacity_hint)
+        return acc
+    end
+end
+
+"""
+    accumulate!(acc::SparseAccumulator, spectrum_idx::Int, bin_indices::AbstractVector{Int},
+                intensities::AbstractVector{Float64})
+
+Appends peak data for one spectrum into the sparse accumulator.
+"""
+@inline function accumulate!(acc::SparseAccumulator, spectrum_idx::Int,
+                             bin_indices::AbstractVector{Int},
+                             intensities::AbstractVector{Float64})
+    n = length(bin_indices)
+    for k in 1:n
+        @inbounds begin
+            push!(acc.I, bin_indices[k])
+            push!(acc.J, spectrum_idx)
+            push!(acc.V, intensities[k])
+        end
+    end
+end
+
+# =============================================================================
+# The Pipeline Executor
+# =============================================================================
+
+"""
+    process_dataset!(data::MSIData, config::PipelineConfig;
+                     progress_callback::Union{Function, Nothing}=nothing,
+                     masked_indices::Union{AbstractVector{Int}, Nothing}=nothing)
+
+The Sprint 2 master streaming function. Processes an entire MSI dataset through
+a kernel chain without holding more than 1 spectrum per thread in RAM.
+
+# Returns
+- `SparseMatrixCSC{Float64, Int}`: The feature matrix (bins × spectra)
+- `Vector{Float64}`: The m/z bin centers
+
+# Architecture
+1. Ensures analytics are computed (for global m/z range)
+2. Creates thread-local SparseAccumulators
+3. Streams spectra via `_iterate_spectra_fast`
+4. Per spectrum: copy view → kernel chain → peak detect → bin → accumulate
+5. Merges accumulators → `sparse(I, J, V)`
+"""
+function process_dataset!(data::MSIData, config::PipelineConfig;
+                          progress_callback::Union{Function, Nothing}=nothing,
+                          masked_indices::Union{AbstractVector{Int}, Nothing}=nothing)
+    
+    # --- Step 1: Ensure analytics are computed (provides global m/z range) ---
+    if !is_set(data.analytics_ready)
+        println("Pre-computing analytics for streaming pipeline...")
+        precompute_analytics(data)
+    end
+    
+    # Determine global m/z range for binning
+    global_min_mz = Base.Threads.atomic_add!(data.global_min_mz, 0.0)
+    global_max_mz = Base.Threads.atomic_add!(data.global_max_mz, 0.0)
+    
+    if !isfinite(global_min_mz) || !isfinite(global_max_mz) || global_min_mz >= global_max_mz
+        @warn "Invalid global m/z range: [$global_min_mz, $global_max_mz]. Cannot bin peaks."
+        return spzeros(0, 0), Float64[]
+    end
+    
+    num_bins = config.num_bins
+    bin_edges = range(global_min_mz, stop=global_max_mz, length=num_bins + 1)
+    bin_centers = [(bin_edges[i] + bin_edges[i+1]) / 2 for i in 1:num_bins]
+    inv_bin_width = 1.0 / step(bin_edges)
+    
+    num_spectra = length(data.spectra_metadata)
+    indices_to_process = masked_indices === nothing ? nothing : masked_indices
+    
+    # --- Step 2: Create thread-local accumulators ---
+    n_threads = Base.Threads.nthreads()
+    accumulators = [SparseAccumulator(num_spectra * 10) for _ in 1:n_threads]
+    spectra_processed = Base.Threads.Atomic{Int}(0)
+    
+    # NEW: Create dedicated workspace buffers for each thread.
+    # This completely eliminates the need for acquire/release and prevents deadlocks.
+    workspaces_mz = [Vector{Float64}(undef, 0) for _ in 1:n_threads]
+    workspaces_int = [Vector{Float64}(undef, 0) for _ in 1:n_threads]
+    workspaces_scratch = [Vector{Float64}(undef, 0) for _ in 1:n_threads]
+    
+    # Pre-parse step configuration for fast dispatch in the hot loop
+    has_smoothing = false
+    has_baseline = false
+    has_normalization = false
+    has_transform = false
+    has_peak_picking = false
+    has_calibration = false
+    
+    smooth_params = Dict{Symbol, Any}()
+    baseline_params = Dict{Symbol, Any}()
+    norm_params = Dict{Symbol, Any}()
+    transform_params = Dict{Symbol, Any}()
+    peak_params = Dict{Symbol, Any}()
+    
+    for s in config.steps
+        if s.name === :smoothing
+            has_smoothing = true
+            smooth_params = s.params
+        elseif s.name === :baseline_correction
+            has_baseline = true
+            baseline_params = s.params
+        elseif s.name === :normalization
+            has_normalization = true
+            norm_params = s.params
+        elseif s.name === :stabilization || s.name === :intensity_transformation
+            has_transform = true
+            transform_params = s.params
+        elseif s.name === :peak_picking
+            has_peak_picking = true
+            peak_params = s.params
+        elseif s.name === :calibration
+            has_calibration = true
+        end
+    end
+    
+    reference_masses = config.reference_peaks
+    
+    # --- Step 3: Stream and process ---
+    start_time = time_ns()
+    
+    # Use let block to capture all variables cleanly for the closure
+    let data=data, accumulators=accumulators, spectra_processed=spectra_processed,
+        bin_edges=bin_edges, num_bins=num_bins, inv_bin_width=inv_bin_width,
+        global_min_mz=global_min_mz,
+        workspaces_mz=workspaces_mz, workspaces_int=workspaces_int, workspaces_scratch=workspaces_scratch,
+        has_smoothing=has_smoothing, has_baseline=has_baseline,
+        has_normalization=has_normalization, has_transform=has_transform,
+        has_peak_picking=has_peak_picking, has_calibration=has_calibration,
+        smooth_params=smooth_params, baseline_params=baseline_params,
+        norm_params=norm_params, transform_params=transform_params,
+        peak_params=peak_params, reference_masses=reference_masses
+        
+        _iterate_spectra_fast(data, indices_to_process) do idx, mz_view, int_view
+            thread_id = Base.Threads.threadid()
+            acc = accumulators[thread_id]
+            
+            # --- Grab Thread-Local Workspaces ---
+            # No locking, no blocking, guaranteed to be available
+            mz_buf = workspaces_mz[thread_id]
+            int_buf = workspaces_int[thread_id]
+            scratch_buf = workspaces_scratch[thread_id]
+            
+            resize!(mz_buf, length(mz_view))
+            resize!(int_buf, length(int_view))
+            resize!(scratch_buf, length(int_view))
+            copyto!(mz_buf, mz_view)
+            copyto!(int_buf, int_view)
+            
+            # --- Kernel Chain (in pipeline order) ---
+                
+                # Category B: Fixed-reference calibration
+                if has_calibration && !isempty(reference_masses)
+                    calibrate_inplace!(mz_buf, int_buf, reference_masses)
+                end
+                
+                # Category A: Intensity transformation
+                if has_transform
+                    transform_inplace!(int_buf, get(transform_params, :method, :sqrt))
+                end
+                
+                # Category A: Smoothing
+                if has_smoothing
+                    smooth_inplace!(int_buf, scratch_buf, data;
+                        method=get(smooth_params, :method, :savitzky_golay),
+                        window=get(smooth_params, :window, 9),
+                        order=get(smooth_params, :order, 2))
+                end
+                
+                # Category A: Baseline correction
+                if has_baseline
+                    baseline_subtract_inplace!(int_buf, scratch_buf, data;
+                        method=get(baseline_params, :method, :snip),
+                        iterations=get(baseline_params, :iterations, 100),
+                        window=get(baseline_params, :window, 20))
+                end
+                
+                # Category A: Normalization
+                if has_normalization
+                    normalize_inplace!(int_buf, get(norm_params, :method, :tic))
+                end
+                
+                # --- Peak Detection & Binning ---
+                if has_peak_picking
+                    detect_peaks_streaming(mz_buf, int_buf, scratch_buf;
+                        method=get(peak_params, :method, :profile),
+                        snr_threshold=Float64(get(peak_params, :snr_threshold, 3.0)),
+                        half_window=Int(get(peak_params, :half_window, 10)),
+                        min_peak_prominence=Float64(get(peak_params, :min_peak_prominence, 0.1)),
+                        merge_peaks_tolerance=Float64(get(peak_params, :merge_peaks_tolerance, 0.002))) do peak_mz, peak_int
+                        
+                        # Bin each discovered peak directly
+                        bin_idx = trunc(Int, (peak_mz - global_min_mz) * inv_bin_width) + 1
+                        bin_idx = clamp(bin_idx, 1, num_bins)
+                        
+                        push!(acc.I, bin_idx)
+                        push!(acc.J, idx)
+                        push!(acc.V, peak_int)
+                    end
+                else
+                    # No peak picking: bin raw intensity directly
+                    @inbounds for i in eachindex(mz_buf)
+                        bin_idx = trunc(Int, (mz_buf[i] - global_min_mz) * inv_bin_width) + 1
+                        bin_idx = clamp(bin_idx, 1, num_bins)
+                        
+                        push!(acc.I, bin_idx)
+                        push!(acc.J, idx)
+                        push!(acc.V, int_buf[i])
+                    end
+                end
+                
+                Base.Threads.atomic_add!(spectra_processed, 1)
+        end
+    end
+    
+    # --- Step 4: Merge thread-local accumulators ---
+    total_entries = sum(length(acc.I) for acc in accumulators)
+    merged_I = Vector{Int}(undef, total_entries)
+    merged_J = Vector{Int}(undef, total_entries)
+    merged_V = Vector{Float64}(undef, total_entries)
+    
+    offset = 0
+    for acc in accumulators
+        n = length(acc.I)
+        if n > 0
+            copyto!(merged_I, offset + 1, acc.I, 1, n)
+            copyto!(merged_J, offset + 1, acc.J, 1, n)
+            copyto!(merged_V, offset + 1, acc.V, 1, n)
+            offset += n
+        end
+    end
+    
+    # --- Step 5: Assemble sparse matrix ---
+    # Use max combiner: when multiple peaks map to the same bin for same spectrum,
+    # keep the maximum intensity
+    feature_matrix = sparse(merged_I, merged_J, merged_V, num_bins, num_spectra, max)
+    
+    # --- Step 6: Apply frequency threshold if configured ---
+    if config.frequency_threshold > 0.0
+        # Count how many spectra have a non-zero value in each bin
+        bin_presence = vec(sum(feature_matrix .> 0, dims=2))
+        min_count = ceil(Int, config.frequency_threshold * num_spectra)
+        keep_bins = findall(bin_presence .>= min_count)
+        feature_matrix = feature_matrix[keep_bins, :]
+        bin_centers = bin_centers[keep_bins]
+    end
+    
+    duration = (time_ns() - start_time) / 1e9
+    n_processed = spectra_processed[]
+    n_nonzeros = nnz(feature_matrix)
+    sparsity = 1.0 - n_nonzeros / (size(feature_matrix, 1) * size(feature_matrix, 2) + 1)
+    
+    @printf "Streaming pipeline complete: %d spectra processed in %.2f seconds.\n" n_processed duration
+    @printf "Feature matrix: %d bins × %d spectra, %d non-zeros (%.1f%% sparse)\n" size(feature_matrix, 1) size(feature_matrix, 2) n_nonzeros sparsity * 100
+    @printf "RAM: %.1f MB (vs %.1f MB dense)\n" (n_nonzeros * 16) / 1e6 (size(feature_matrix, 1) * size(feature_matrix, 2) * 8) / 1e6
+    
+    if progress_callback !== nothing
+        progress_callback(1.0)
+    end
+    
+    return feature_matrix, collect(Float64, bin_centers)
+end
+
+"""
+    save_sparse_matrix(matrix::SparseMatrixCSC, output_path::String)
+
+Exports a highly optimized SparseMatrixCSC array to disk using the standard
+Matrix Market Coordinate format (`.mtx`), guaranteeing no bottleneck or OOM crashes 
+for extremely large MS dataset persistence.
+"""
+function save_sparse_matrix(matrix::SparseMatrixCSC{Float64, Int}, output_path::String)
+    m, n = size(matrix)
+    nnz_val = nnz(matrix)
+    # Use streaming I/O with a large buffer for ultra-fast persistence
+    open(output_path, "w") do io
+        # Write Matrix Market Header
+        write(io, "%%MatrixMarket matrix coordinate real general\n")
+        write(io, "$m $n $nnz_val\n")
+        
+        # Directly extract CSC properties (O(1) memory, zero allocation)
+        row_indices = rowvals(matrix)
+        values_array = nonzeros(matrix)
+        
+        @inbounds for filter_j in 1:n
+            # nzrange returns the index bounds for non-zero elements in column 'j'
+            for idx in nzrange(matrix, filter_j)
+                i = row_indices[idx]
+                v = values_array[idx]
+                write(io, "$i $filter_j $v\n")
+            end
+        end
+    end
+end

src/mzML.jl

@@ -19,7 +19,26 @@ const DATA_FORMAT_ACCESSIONS = Dict{String, DataType}(
     "MS:1000523" => Float64
 )
 
+"""
+    parse_instrument_metadata_mzml(stream::IO)
+
+Parses instrument metadata from an mzML file.
+
+# Arguments
+
+- `stream::IO`: The stream to parse the metadata from.
+
+# Returns
+- `InstrumentMetadata`: The instrument metadata.
+
+# Example
+
+```julia
+parse_instrument_metadata_mzml(open("test.mzML"))
+```
+"""
 function parse_instrument_metadata_mzml(stream::IO)
+    println("DEBUG: Starting mzML instrument metadata parsing...")
     # Initialize with default values from the InstrumentMetadata constructor
     instrument_meta = InstrumentMetadata()
     
@@ -67,37 +86,53 @@ function parse_instrument_metadata_mzml(stream::IO)
 
                     if acc == "MS:1000031" # instrument model
                         instrument_model = val
+                        #println("DEBUG: Instrument model: $instrument_model")
                     elseif acc == "MS:1001496" # mass resolving power (more specific)
                         resolution = tryparse(Float64, val)
+                        #println("DEBUG: Resolution (specific): $resolution")
                     elseif acc == "MS:1000011" && resolution === nothing # resolution (less specific)
                         resolution = tryparse(Float64, val)
+                        #println("DEBUG: Resolution (less specific): $resolution")
                     elseif acc == "MS:1000016" # mass accuracy (ppm)
                         mass_accuracy_ppm = tryparse(Float64, val)
+                        #println("DEBUG: Mass accuracy (ppm): $mass_accuracy_ppm")
                     elseif acc == "MS:1000130" # positive scan
                         polarity = :positive
+                        #println("DEBUG: Polarity: positive")
                     elseif acc == "MS:1000129" # negative scan
                         polarity = :negative
+                        #println("DEBUG: Polarity: negative")
                     elseif acc == "MS:1000592" # external calibration
                         calibration_status = :external
+                        #println("DEBUG: Calibration status: external")
                     elseif acc == "MS:1000593" # internal calibration
                         calibration_status = :internal
+                        #println("DEBUG: Calibration status: internal")
                     elseif acc == "MS:1000747" && calibration_status == :uncalibrated # instrument specific calibration
                         calibration_status = :internal # Assume as a form of internal calibration
+                        #println("DEBUG: Calibration status: instrument specific (internal)")
                     elseif acc == "MS:1000867" # laser wavelength
                         laser_settings["wavelength_nm"] = tryparse(Float64, val)
+                        #println("DEBUG: Laser wavelength: $(laser_settings["wavelength_nm"]) nm")
                     elseif acc == "MS:1000868" # laser fluence
                         laser_settings["fluence"] = tryparse(Float64, val)
+                        #println("DEBUG: Laser fluence: $(laser_settings["fluence"])")
                     elseif acc == "MS:1000869" # laser repetition rate
                         laser_settings["repetition_rate_hz"] = tryparse(Float64, val)
+                        #println("DEBUG: Laser repetition rate: $(laser_settings["repetition_rate_hz"]) Hz")
                     # NEW: Vendor Preprocessing terms
                     elseif acc == "MS:1000579" # baseline correction
                         push!(vendor_preprocessing_steps, "Baseline Correction")
+                        #println("DEBUG: Vendor preprocessing step: Baseline Correction")
                     elseif acc == "MS:1000580" # smoothing
                         push!(vendor_preprocessing_steps, "Smoothing")
+                        #println("DEBUG: Vendor preprocessing step: Smoothing")
                     elseif acc == "MS:1000578" # data transformation (e.g., centroiding)
                         push!(vendor_preprocessing_steps, "Data Transformation: $(name)")
+                        #println("DEBUG: Vendor preprocessing step: Data Transformation: $(name)")
                     elseif acc == "MS:1000800" # deisotoping
                         push!(vendor_preprocessing_steps, "Deisotoping")
+                        #println("DEBUG: Vendor preprocessing step: Deisotoping")
                     end
                 end
             end
@@ -106,6 +141,8 @@ function parse_instrument_metadata_mzml(stream::IO)
         @warn "Could not fully parse instrument metadata from mzML header. Using defaults. Error: $e"
     end
     
+    println("DEBUG: Finished mzML instrument metadata parsing.")
+    
     # Always return a valid object
     return InstrumentMetadata(
         resolution,
@@ -137,14 +174,15 @@ determine the data type, compression, and axis type.
 function get_spectrum_asset_metadata(stream::IO)
     start_pos = position(stream)
     
+    #println("DEBUG: Entering get_spectrum_asset_metadata to parse binaryDataArray...")
+
     bda_tag = find_tag(stream, r"<binaryDataArray\s+encodedLength=\"(\d+)\"")
 
     if bda_tag === nothing
-    
         throw(FileFormatError("Cannot find binaryDataArray"))
-    
     end
     encoded_length = parse(Int32, bda_tag.captures[1])
+    #println("DEBUG:   Encoded length: $encoded_length")
 
     # Initialize parameters as separate variables with concrete types
     data_format::DataType = Float64
@@ -166,14 +204,19 @@ function get_spectrum_asset_metadata(stream::IO)
             # Use constant comparisons and dictionary lookup for better performance
             if acc_str == MZ_AXIS_ACCESSION
                 axis = :mz
+                #println("DEBUG:   Axis type identified as: m/z")
             elseif acc_str == INTENSITY_AXIS_ACCESSION
                 axis = :intensity
+                #println("DEBUG:   Axis type identified as: intensity")
             elseif haskey(DATA_FORMAT_ACCESSIONS, acc_str)
                 data_format = DATA_FORMAT_ACCESSIONS[acc_str]
+                #println("DEBUG:   Data format identified as: $data_format")
             elseif acc_str == COMPRESSION_ACCESSION
                 compression_flag = true
+                #println("DEBUG:   Compression: true")
             elseif acc_str == NO_COMPRESSION_ACCESSION
                 compression_flag = false
+                #println("DEBUG:   Compression: false")
             end
         end
     end
@@ -181,12 +224,14 @@ function get_spectrum_asset_metadata(stream::IO)
     seek(stream, start_pos)
     readuntil(stream, "<binary>")
     binary_offset = position(stream)
+    #println("DEBUG:   Binary data offset: $binary_offset")
 
     # Move stream to the end of the binary data array for the next iteration
     readuntil(stream, "</binaryDataArray>")
+    #println("DEBUG: Exiting get_spectrum_asset_metadata.")
 
     # Create SpectrumAsset directly from the variables
-    return SpectrumAsset(data_format, compression_flag, binary_offset, encoded_length, axis)
+    return SpectrumAsset(data_format, compression_flag, binary_offset, encoded_length, axis, 0.0, 0.0)
 end
 
 # This function is updated to return the generic SpectrumMetadata struct
@@ -223,13 +268,16 @@ function parse_spectrum_metadata(stream::IO, offset::Int64)
 
     id_match = match(r"<spectrum\s+index=\"\d+\"\s+id=\"([^\"]+)", spectrum_xml)
     id = id_match === nothing ? "" : id_match.captures[1]
+    #println("DEBUG:   Parsing spectrum ID: $id")
 
     # Determine mode from the XML block
     mode = UNKNOWN
     if occursin("MS:1000127", spectrum_xml)
         mode = CENTROID
+        #println("DEBUG:   Spectrum mode: CENTROID")
     elseif occursin("MS:1000128", spectrum_xml)
         mode = PROFILE
+        #println("DEBUG:   Spectrum mode: PROFILE")
     end
 
     # Find where the binary data list starts to parse assets
@@ -248,6 +296,10 @@ function parse_spectrum_metadata(stream::IO, offset::Int64)
         (asset2, asset1)
     end
 
+    #println("DEBUG:     m/z Asset - Format: $(mz_asset.format), Compressed: $(mz_asset.is_compressed), Offset: $(mz_asset.offset), Encoded Length: $(mz_asset.encoded_length)")
+    #println("DEBUG:     Intensity Asset - Format: $(int_asset.format), Compressed: $(int_asset.is_compressed), Offset: $(int_asset.offset), Encoded Length: $(int_asset.encoded_length)")
+    #println("DEBUG:   Finished parsing spectrum ID: $id metadata.")
+
     # Create the new unified metadata object
     # For mzML, x and y coordinates are not applicable, so we use 0.
     return SpectrumMetadata(Int32(0), Int32(0), id, :sample, mode, mz_asset, int_asset)
@@ -294,6 +346,19 @@ end
 
 Finds the index offset in an mzML file by reading from the end.
 Optimized version with better buffer management.
+
+# Arguments
+
+- `stream::IO`: The stream to parse the metadata from.
+
+# Returns
+- `Int64`: The index offset.
+
+# Example
+
+```julia
+find_index_offset(open("test.mzML"))
+```
 """
 function find_index_offset(stream::IO)::Int64
     file_size = filesize(stream)
@@ -329,38 +394,48 @@ then parses the metadata for each spectrum without loading the binary data.
 """
 function load_mzml_lazy(file_path::String; cache_size::Int=100)
     println("DEBUG: Opening file stream for $file_path")
-    ts_stream = ThreadSafeFileHandle(file_path, "r")
+    
+    # --- Handle Pool Optimization ---
+    # Open multiple handles to the .mzML file to avoid lock contention
+    num_handles = Threads.nthreads()
+    mzml_handles = [open(file_path, "r") for _ in 1:num_handles]
+    
+    # Use the first handle for initial parsing
+    primary_handle = mzml_handles[1]
 
     try
         # --- NEW: Parse instrument metadata from header ---
         println("DEBUG: Parsing instrument metadata from header...")
-        instrument_meta = parse_instrument_metadata_mzml(ts_stream.handle)
+        instrument_meta = parse_instrument_metadata_mzml(primary_handle)
 
-        println("--- Extracted Instrument Metadata ---")
+        seekstart(primary_handle) # Reset stream after header parsing
-        println("Resolution: ", instrument_meta.resolution)
-        println("Acquisition Mode (pre-check): ", instrument_meta.acquisition_mode)
-        println("Calibration Status: ", instrument_meta.calibration_status)
-        println("Instrument Model: ", instrument_meta.instrument_model)
-        println("Mass Accuracy (ppm): ", instrument_meta.mass_accuracy_ppm)
-        println("Laser Settings: ", instrument_meta.laser_settings)
-        println("Polarity: ", instrument_meta.polarity)
-        println("------------------------------------")
-
-        seekstart(ts_stream.handle) # Reset stream after header parsing
 
         println("DEBUG: Finding index offset...")
-        index_offset = find_index_offset(ts_stream.handle)
+        index_offset = find_index_offset(primary_handle)
+        
+        # --- NEW: Mmap Optimization with RAM Safety ---
+        mmap_data = nothing
+        try
+            file_size = filesize(file_path)
+
+            println("DEBUG: Memory mapping .mzML file...")
+            seekstart(primary_handle) # Anchor Mmap to the beginning of the file to prevent overflow
+            mmap_data = Mmap.mmap(primary_handle, Vector{UInt8}, (file_size,))
+            println("DEBUG: .mzML file mmapped successfully.")
+        catch e
+            @warn "Memory mapping failed for mzML, falling back to standard I/O: $e"
+        end
+
         println("DEBUG: Seeking to index list at offset $index_offset.")
-        seek(ts_stream.handle, index_offset)
+        seek(primary_handle, index_offset)
         
         println("DEBUG: Searching for '<index name=\"spectrum\">'.")
-        if find_tag(ts_stream.handle, r"<index\s+name=\"spectrum\"") === nothing
+        if find_tag(primary_handle, r"<index\s+name=\"spectrum\"") === nothing
             throw(FileFormatError("Could not find spectrum index."))
         end
-        println("DEBUG: Found spectrum index tag.")
 
         println("DEBUG: Parsing spectrum offsets...")
-        spectrum_offsets = parse_offset_list(ts_stream.handle)
+        spectrum_offsets = parse_offset_list(primary_handle)
         if isempty(spectrum_offsets)
             throw(FileFormatError("No spectrum offsets found."))
         end
@@ -368,29 +443,25 @@ function load_mzml_lazy(file_path::String; cache_size::Int=100)
         println("DEBUG: Found $num_spectra spectrum offsets.")
 
         println("DEBUG: Parsing metadata for each spectrum...")
-        # Pre-allocate the metadata vector for better performance
         spectra_metadata = Vector{SpectrumMetadata}(undef, num_spectra)
         
-        # Use @inbounds for faster indexing in the loop
         @inbounds for i in 1:num_spectra
-            spectra_metadata[i] = parse_spectrum_metadata(ts_stream.handle, spectrum_offsets[i])
+            spectra_metadata[i] = parse_spectrum_metadata(primary_handle, spectrum_offsets[i])
             
-            # Progress reporting for large files
             if i % 1000 == 0
                 println("DEBUG: Processed $i/$num_spectra spectra")
             end
         end
-        println("DEBUG: Metadata parsing complete.")
+        println("DEBUG: Metadata parsing complete for all $num_spectra spectra.")
 
-        # Assuming uniform data formats, take from the first spectrum
+        # Inferred global formats from first spectrum
         first_meta = spectra_metadata[1]
         mz_format = first_meta.mz_asset.format
         intensity_format = first_meta.int_asset.format
         
-        # --- NEW: Determine overall acquisition mode ---
+        # Determine overall acquisition mode ...
-        modes = [meta.mode for meta in spectra_metadata]
+        num_centroid = count(m -> m.mode == CENTROID, spectra_metadata)
-        num_centroid = count(m -> m == CENTROID, modes)
+        num_profile = count(m -> m.mode == PROFILE, spectra_metadata)
-        num_profile = count(m -> m == PROFILE, modes)
         
         acq_mode_symbol = if num_centroid > 0 && num_profile == 0
             :centroid
@@ -404,146 +475,25 @@ function load_mzml_lazy(file_path::String; cache_size::Int=100)
 
         final_instrument_meta = InstrumentMetadata(
             instrument_meta.resolution,
-            acq_mode_symbol, # Update with parsed mode
+            acq_mode_symbol, 
             instrument_meta.mz_axis_type,
             instrument_meta.calibration_status,
             instrument_meta.instrument_model,
             instrument_meta.mass_accuracy_ppm,
             instrument_meta.laser_settings,
             instrument_meta.polarity,
-            instrument_meta.vendor_preprocessing_steps # Add this new field
+            instrument_meta.vendor_preprocessing_steps
         )
 
-        source = MzMLSource(ts_stream, mz_format, intensity_format)
+        source = MzMLSource(mzml_handles, mz_format, intensity_format, mmap_data)
         println("DEBUG: Creating MSIData object.")
         return MSIData(source, spectra_metadata, final_instrument_meta, (0, 0), nothing, cache_size)
 
     catch e
-        close(ts_stream) # Ensure stream is closed on error
+        # Close all handles in the pool if initialization fails
+        for h in mzml_handles
+            isopen(h) && close(h)
+        end
         rethrow(e)
     end
 end
-
-#=
-"""
-    LoadMzml(fileName::String)
-
-Eagerly loads all spectra from a .mzML file into memory.
-This function now uses the new lazy-loading
-architecture internally but presents the data in the old format.
-
-# Arguments
-- `fileName`: The path to the `.mzML` file.
-
-# Returns
-- A `2xN` matrix where `N` is the number of spectra. The first row contains
-  m/z arrays and the second row contains intensity arrays.
-"""
-function LoadMzml(fileName::String)
-    # Use the lazy loader to get the MSIData object
-    msi_data = load_mzml_lazy(fileName, cache_size=0) # No need to cache if we load all
-    
-    num_spectra = length(msi_data.spectra_metadata)
-    
-    # FIXED: Use concrete typed arrays instead of Array{Any}
-    spectra_matrix = Vector{Tuple{Vector{Float64}, Vector{Float64}}}(undef, num_spectra)
-    
-    # Pre-allocate and use bounds checking optimization
-    @inbounds for i in 1:num_spectra
-        # Use the new GetSpectrum API
-        mz, intensity = GetSpectrum(msi_data, i)
-        spectra_matrix[i] = (mz, intensity)
-    end
-    
-    # Convert to the expected 2xN format if needed by downstream code
-    # Note: This maintains the original interface but with better typing
-    result_matrix = Array{Any}(undef, (2, num_spectra))
-    @inbounds for i in 1:num_spectra
-        result_matrix[1, i] = spectra_matrix[i][1]
-        result_matrix[2, i] = spectra_matrix[i][2]
-    end
-    
-    return result_matrix
-end
-=#
-
-#=
-"""
-    load_mzml_batch(file_path::String, spectrum_indices::AbstractVector{Int})
-
-Loads a specific batch of spectra from an mzML file efficiently.
-Useful for parallel processing or when only specific spectra are needed.
-
-# Arguments
-- `file_path`: Path to the mzML file
-- `spectrum_indices`: Indices of spectra to load
-
-# Returns
-- Vector of (mz_array, intensity_array) tuples
-"""
-function load_mzml_batch(file_path::String, spectrum_indices::AbstractVector{Int})
-    msi_data = load_mzml_lazy(file_path)
-    num_to_load = length(spectrum_indices)
-    
-    # Pre-allocate result with concrete types
-    results = Vector{Tuple{Vector{Float64}, Vector{Float64}}}(undef, num_to_load)
-    
-    @inbounds for (i, idx) in enumerate(spectrum_indices)
-        mz, intensity = GetSpectrum(msi_data, idx)
-        results[i] = (mz, intensity)
-    end
-    
-    return results
-end
-=#
-
-#=
-"""
-    get_mzml_summary(file_path::String)::NamedTuple
-
-Quickly extracts summary information from an mzML file without loading all metadata.
-
-# Returns
-- Named tuple with: num_spectra, mz_range, intensity_range, data_formats
-"""
-function get_mzml_summary(file_path::String)::NamedTuple
-    ts_stream = ThreadSafeFileHandle(file_path, "r")
-    try
-        index_offset = find_index_offset(ts_stream.handle)
-        seek(ts_stream.handle, index_offset)
-        
-        if find_tag(ts_stream.handle, r"<index\s+name=\"spectrum\"") === nothing
-            throw(FileFormatError("Could not find spectrum index."))
-        end
-        
-        spectrum_offsets = parse_offset_list(ts_stream.handle)
-        num_spectra = length(spectrum_offsets)
-        
-        # Sample first few spectra to determine formats and ranges
-        sample_size = min(10, num_spectra)
-        sample_metadata = [parse_spectrum_metadata(ts_stream.handle, spectrum_offsets[i]) for i in 1:sample_size]
-        
-        # Extract formats from sample
-        mz_format = sample_metadata[1].mz_asset.format
-        intensity_format = sample_metadata[1].int_asset.format
-        
-        return (num_spectra=num_spectra, 
-                mz_format=mz_format, 
-                intensity_format=intensity_format,
-                sample_spectra=sample_size)
-    finally
-        close(ts_stream)
-    end
-end
-=#
-
-#=
-# Performance optimization: Cache frequently used regex patterns
-const PRE_COMPILED_REGEX = (
-    encoded_length = r"<binaryDataArray\s+encodedLength=\"(\d+)\"",
-    spectrum_id = r"<spectrum\s+index=\"\d+\"\s+id=\"([^\"]+)",
-    offset = r"<offset[^>]*>(\d+)</offset>",
-    index_list = r"<index\s+name=\"spectrum\"",
-    index_offset = r"<indexListOffset>(\d+)</indexListOffset>"
-)
-=#

start_MSI_GUI.jl

@@ -1,16 +1,70 @@
 using Pkg
 sTime=time()
 Pkg.activate(".")
-# Only instantiate in development mode
+ENV["GENIE_ENV"] = "dev"
-if get(ENV, "GENIE_ENV", "dev") != "prod"
+#ENV["GENIE_ENV"] = "prod"
-    @info "Development environment detected. Instantiating packages..."
+
+manifest_path = joinpath(@__DIR__, "Manifest.toml")
+
+# Selective instantiation for faster startup
+if get(ENV, "GENIE_ENV", "dev") != "prod" && !isfile(manifest_path)
+    @info "Development environment detected and Manifest.toml missing. Instantiating packages..."
     Pkg.resolve()
     Pkg.instantiate()
+    Pkg.gc()
+elseif get(ENV, "GENIE_ENV", "dev") != "prod"
+    @info "Manifest.toml found. Skipping Pkg.instantiate() for faster boot. Delete Manifest.toml if you need to re-instantiate."
+    ENV["GENIE_ENV"] = "prod"
 end
-Pkg.gc()
 
 using Genie
 
+# --- Cross-Platform Startup Cleanup ---
+# Remove orphaned GenieSessionFileSession directories from previous runs.
+# These accumulate in the OS temp directory as jl_XXXXXX folders containing
+# serialized session files (64-char hex filenames). Over long sessions or
+# after crashes, they can consume gigabytes of disk space.
+function cleanup_orphaned_sessions()
+    tmp = Base.tempdir()
+    cleaned_count = 0
+    cleaned_bytes = 0
+
+    for entry in readdir(tmp; join=false)
+        # Only target directories matching Julia's temp naming pattern
+        startswith(entry, "jl_") || continue
+        full_path = joinpath(tmp, entry)
+        isdir(full_path) || continue
+
+        # Validate: a Genie session dir contains files with 64-char hex names
+        try
+            contents = readdir(full_path)
+            isempty(contents) && continue
+
+            # Check if at least one file matches the 64-char hex session ID pattern
+            is_session_dir = any(contents) do f
+                length(f) == 64 && all(c -> c in "0123456789abcdef", f)
+            end
+            is_session_dir || continue
+
+            # Safe to remove — this is an orphaned Genie session directory
+            dir_size = sum(filesize(joinpath(full_path, f)) for f in contents; init=0)
+            rm(full_path; recursive=true, force=true)
+            cleaned_count += 1
+            cleaned_bytes += dir_size
+        catch e
+            @debug "Skipping $entry during cleanup: $e"
+        end
+    end
+
+    if cleaned_count > 0
+        size_mb = round(cleaned_bytes / (1024^2), digits=1)
+        @info "Startup cleanup: removed $cleaned_count orphaned session dir(s), freed $(size_mb) MB"
+    end
+end
+
+cleanup_orphaned_sessions()
+
+
 # Load and configure Genie
 Genie.loadapp()
 

test/benchmark.jl

@@ -0,0 +1,936 @@
+# test/benchmark.jl
+
+# ===================================================================
+# Performance Benchmark Suite for JuliaMSI
+# ===================================================================
+# This script compares the performance of the new `JuliaMSI` pipeline
+# against the old `julia_mzML_imzML` library.
+#
+# It measures the time and memory allocation for loading an `.imzML`
+# file and generating an image slice multiple times.
+#
+# Instructions:
+# 1. Fill in the placeholder paths in the "CONFIG" section below.
+# 2. Run the script from the project's root directory:
+#    julia --project=. test/benchmark.jl
+# 3. Check `test/results/` for `benchmark_results.csv` and plots.
+# ===================================================================
+
+using Pkg
+Pkg.activate(joinpath(@__DIR__, "..")) # Activate main project environment
+
+using DataFrames
+using CSV
+using CairoMakie
+using ColorSchemes
+using Statistics
+using Printf
+using JSON
+using MSI_src # Import the new library
+using .MSI_src: get_mz_slice, quantize_intensity, save_bitmap, get_multiple_mz_slices
+using julia_mzML_imzML
+
+# Remember, to run this, you must enter pkg mode in julia and add this library:
+# add github.com/CINVESTAV-LABI/julia_mzML_imzML
+
+# ===================================================================
+# CONFIG: PLEASE FILL IN YOUR FILE PATHS AND PARAMETERS
+# ===================================================================
+
+struct BenchmarkCase
+    filepath::String
+    mz_value::Float64
+    mz_tolerance::Float64
+    name::String # New field for a descriptive name
+end
+
+const BENCHMARK_CASES = [
+    # Add your benchmark cases here. Example:
+    # BenchmarkCase("/path/to/your/small_file.imzML", 309.06, 0.1),
+    # BenchmarkCase("/path/to/your/medium_file.imzML", 896.0, 1.0),
+    # BenchmarkCase("/path/to/your/large_file.imzML", 100.0, 0.1),
+    # BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_LA-ESI/180817_NEG_Thaliana_Leaf_bottom_1_0841.imzML",116.07,0.1, "Thaliana Leaf"),
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/Chilli/ltpmsi-chilli.imzML",420,0.1, "Chilli Pepper"), # Chilli
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_DESI/40TopL,10TopR,30BottomL,20BottomR/40TopL,10TopR,30BottomL,20BottomR-centroid.imzML",885.5,0.1, "Colon Cancer Human"), # Human Cancer
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML", 716.053,0.1, "Mouse Urinary Bladder"), # Mouse bladder
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/Leafs/CE1_Leaf_R3.imzML",306.1,0.1, "Leaf"),
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/Liv2_imzML_TIMSConvert-selected/Liv2.imzML",796.18,0.1, "Liver Cut"), #Lib2
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML",804.3,0.1, "Mouse Stomach 2GB"), # Mouse Stomach
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/salida_Tims/Stomach_DHB.imzML",804.3,0.1, "Mouse Stomach 4GB"), # Mouse Stomach
+    
+
+]
+
+const NUM_REPETITIONS = 50 # Number of times to generate the image for averaging
+const BULK_MZ_VARIATIONS = 50 # Number of m/z variations to generate for the bulk benchmark (e.g., mz, mz+0.1, ..., mz+(N-1)*0.1)
+const RESULTS_DIR = joinpath(@__DIR__, "results")
+
+# Helper function to generate m/z variations for bulk processing
+function get_mz_variation_vector(base_mz::Float64, num_variations::Int)
+    return [base_mz + i * 0.1 for i in 0:(num_variations - 1)]
+end
+
+# Helper function to get the combined size of .imzML and .ibd files
+function get_total_file_size_mb(filepath::String)
+    imzml_size = isfile(filepath) ? filesize(filepath) : 0
+    ibd_path = replace(filepath, r"\.(imzML|imzml)$" => ".ibd")
+    ibd_size = isfile(ibd_path) ? filesize(ibd_path) : 0
+    return round((imzml_size + ibd_size) / 1024^2, digits=2)
+end
+
+# ===================================================================
+# BENCHMARK: JuliaMSI (New Library)
+# ===================================================================
+
+"""
+    benchmark_julianew(case::BenchmarkCase)
+
+Runs the benchmark for the modern JuliaMSI library.
+"""
+function benchmark_julianew(case::BenchmarkCase)
+    println("--- Benchmarking JuliaMSI (New) on $(basename(case.filepath)) ---")
+    
+    # 1. Load the data
+    load_stats = @timed OpenMSIData(case.filepath)
+    msi_data = load_stats.value
+    load_time = load_stats.time
+    load_mem = load_stats.bytes / 1024^2 # Convert to MB
+
+    image_times = []
+    image_mems = []
+    local final_image # Declare final_image outside the loop
+
+    # 2. Generate image multiple times
+    for i in 1:NUM_REPETITIONS
+        print("\rRepetition $i/$NUM_REPETITIONS...")
+        image_stats = @timed get_mz_slice(msi_data, case.mz_value, case.mz_tolerance)
+        final_image = image_stats.value
+        push!(image_times, image_stats.time)
+        push!(image_mems, image_stats.bytes / 1024^2)
+    end
+    println("\nDone.")
+
+    # 3. Save one resulting image for validation
+    output_path_bitmap = joinpath(RESULTS_DIR, "benchmark_JuliaMSI_$(basename(case.filepath)).bmp")
+    # Quantize and save the image
+    quantized_image, _ = quantize_intensity(final_image, 20) # 256 levels
+    save_bitmap(output_path_bitmap, quantized_image, MSI_src.ViridisPalette)
+    println("Saved validation image to $output_path_bitmap")
+
+    # 4. Calculate statistics
+    println("DEBUG (New): NUM_REPETITIONS = $NUM_REPETITIONS")
+    println("DEBUG (New): load_time = $load_time")
+    println("DEBUG (New): image_times = $(image_times)")
+    println("DEBUG (New): length(image_times) = $(length(image_times))")
+    avg_image_time = mean(image_times)
+    avg_image_mem = mean(image_mems) # Added missing calculation
+    println("DEBUG (New): avg_image_time = $avg_image_time")
+    println("DEBUG (New): sum(image_times) = $(sum(image_times))")
+    total_time = load_time + sum(image_times)
+    println("DEBUG (New): calculated total_time = $total_time")
+
+    return (
+        library="JuliaMSI",
+        filepath=basename(case.filepath),
+        file_size_mb=get_total_file_size_mb(case.filepath), # Use helper function
+        load_time_s=load_time,
+        avg_image_time_s=avg_image_time,
+        total_time_s=total_time,
+        load_memory_mb=load_mem,
+        avg_image_memory_mb=avg_image_mem,
+        name=case.name # Added case name to results
+    )
+end
+
+"""
+    benchmark_julianew_bulk(case::BenchmarkCase)
+
+Runs the benchmark for the modern JuliaMSI library's bulk slice generation.
+"""
+function benchmark_julianew_bulk(case::BenchmarkCase)
+    println("--- Benchmarking JuliaMSI (Bulk) on $(basename(case.filepath)) ---")
+
+    # Generate multiple m/z values for bulk processing
+    mz_values_for_bulk = get_mz_variation_vector(case.mz_value, BULK_MZ_VARIATIONS)
+    println("  Generating $(length(mz_values_for_bulk)) m/z variations: $(mz_values_for_bulk)")
+    
+    # 1. Load the data
+    load_stats = @timed OpenMSIData(case.filepath)
+    msi_data = load_stats.value
+    load_time = load_stats.time
+    load_mem = load_stats.bytes / 1024^2 # Convert to MB
+
+    image_times = []
+    image_mems = []
+    local final_slices_dict # Declare outside the loop
+
+    # 2. Generate multiple images in bulk (repeated for averaging)
+    for i in 1:1
+        print("\rRepetition $i/$NUM_REPETITIONS for bulk...")
+        # Assume MSI_src.get_multiple_mz_slices exists and returns Dict{Real, Matrix{Float64}}
+        image_stats = @timed MSI_src.get_multiple_mz_slices(msi_data, mz_values_for_bulk, case.mz_tolerance)
+        final_slices_dict = image_stats.value
+        push!(image_times, image_stats.time)
+        
+        # Calculate memory for all slices combined
+        total_slices_mem = sum(Base.summarysize(slice) for slice in values(final_slices_dict)) / 1024^2
+        push!(image_mems, total_slices_mem)
+    end
+    println("\nDone.")
+
+    # No validation image saving for bulk due to multiple slices
+
+    # 3. Calculate statistics
+    avg_image_time = mean(image_times)
+    avg_image_mem = mean(image_mems)
+    total_time = load_time + sum(image_times) # Total time includes all repetitions
+
+    return (
+        library="JuliaMSIBulk",
+        filepath=basename(case.filepath),
+        file_size_mb=get_total_file_size_mb(case.filepath), # Use helper function
+        load_time_s=load_time,
+        avg_image_time_s=avg_image_time,
+        total_time_s=total_time,
+        load_memory_mb=load_mem,
+        avg_image_memory_mb=avg_image_mem,
+        name=case.name # Added case name to results
+    )
+end
+
+
+"""
+    benchmark_juliaold(case::BenchmarkCase)
+
+Runs the benchmark for the old library using dynamic environment switching.
+"""
+function benchmark_juliaold(case::BenchmarkCase)
+    println("--- Benchmarking julia_mzML_imzML (Old) on $(basename(case.filepath)) ---") 
+    try
+        # --- 1. Load Data ---
+        load_stats = @timed LoadImzml(case.filepath)
+        msi_data = load_stats.value
+        load_time = load_stats.time
+        load_mem = load_stats.bytes / 1024^2 # Convert to MB
+
+        image_times = []
+        image_mems = []
+        local last_image_stats_value # Declare outside the loop
+
+        # --- 2. Generate Image Repeatedly ---
+        for i in 1:NUM_REPETITIONS
+            print("\rRepetition $i/$NUM_REPETITIONS...")
+            image_stats = @timed GetSlice(msi_data, case.mz_value, case.mz_tolerance)
+            last_image_stats_value = image_stats.value # Store the value
+            push!(image_times, image_stats.time)
+            push!(image_mems, image_stats.bytes / 1024^2)
+        end
+        println("\nDone.")
+
+        # --- 3. Save Validation Image ---
+        final_image = last_image_stats_value
+        output_path_bitmap = joinpath(RESULTS_DIR, "benchmark_old_lib_$(basename(case.filepath)).bmp")
+        SaveBitmap(output_path_bitmap, IntQuant(final_image), ViridisPalette)
+        println("Saved validation image to $output_path_bitmap")
+
+            # --- 4. Calculate Averages and Totals ---
+            println("DEBUG (Old): NUM_REPETITIONS = $NUM_REPETITIONS")
+            println("DEBUG (Old): load_time = $load_time")
+            println("DEBUG (Old): image_times = $(image_times)")
+            println("DEBUG (Old): length(image_times) = $(length(image_times))")
+            avg_image_time = mean(image_times)
+            println("DEBUG (Old): avg_image_time = $avg_image_time")
+            println("DEBUG (Old): sum(image_times) = $(sum(image_times))")
+            total_time = load_time + sum(image_times)
+            println("DEBUG (Old): calculated total_time = $total_time")
+            avg_image_mem = mean(image_mems) # Added missing calculation
+        return (
+            library="julia_mzML_imzML",
+            filepath=basename(case.filepath),
+            file_size_mb=get_total_file_size_mb(case.filepath), # Use helper function
+            load_time_s=load_time,
+            avg_image_time_s=avg_image_time,
+            total_time_s=total_time,
+        load_memory_mb=load_mem,
+        avg_image_memory_mb=avg_image_mem,
+        name=case.name # Added case name to results
+    )
+        
+    catch e
+        println("ERROR benchmarking old library: $e")
+        println("Make sure julia_mzML_imzML is installed and OLD_LIB_ENV_PATH is set correctly.")
+        return nothing
+    end
+end
+
+function benchmark_multi_file_walkthrough(cases::Vector{BenchmarkCase})
+    println("--- Multi-File Walkthrough Benchmark ---")
+    println("Simulating workflow: load file -> generate 1 image -> close -> next file")
+    
+    results = []
+    
+    for case in cases
+        if !isfile(case.filepath)
+            @warn "File not found: $(case.filepath). Skipping."
+            continue
+        end
+        
+        println("\nProcessing: $(case.name)")
+        
+        # --- NEW LIBRARY (JuliaMSI) ---
+        println("  [JuliaMSI]")
+        # Time includes metadata loading + single image generation
+        new_total_stats = @timed begin
+            data = OpenMSIData(case.filepath)
+            img = get_mz_slice(data, case.mz_value, case.mz_tolerance)
+            close(data) # Explicitly release resources
+        end
+        
+        # --- OLD LIBRARY (julia_mzML_imzML) ---
+        println("  [julia_mzML_imzML]")
+        old_total_stats = try
+            @timed begin
+                data = LoadImzml(case.filepath) # Loads ENTIRE dataset into RAM
+                img = GetSlice(data, case.mz_value, case.mz_tolerance)
+                # Note: No explicit close function in old library
+            end
+        catch e
+            @warn "Old library failed on $(basename(case.filepath)): $e"
+            (time=Inf, bytes=Inf, value=nothing)
+        end
+        
+        push!(results, (
+            name=case.name,
+            file_size_mb=get_total_file_size_mb(case.filepath),
+            julianew_time_s=new_total_stats.time,
+            julianew_memory_mb=new_total_stats.bytes/1024^2,
+            juliaold_time_s=old_total_stats.time,
+            juliaold_memory_mb=old_total_stats.bytes/1024^2,
+            speedup=old_total_stats.time / new_total_stats.time
+        ))
+        
+        # Force garbage collection between files to level the playing field
+        GC.gc()
+        if Sys.islinux()
+            ccall(:malloc_trim, Int32, (Int32,), 0)
+        end
+    end
+    
+    return results
+end
+
+function benchmark_real_world_pipeline(cases::Vector{BenchmarkCase})
+    println("--- REAL-WORLD PIPELINE BENCHMARK ---")
+    println("Simulating: Load → Multiple Slices → Process → Save → Next File")
+    
+    # Realistic m/z values (similar to your UI)
+    mz_values = [116.07, 420.0, 885.5, 716.053, 796.18, 804.3]
+    tolerance = 0.1
+    results = []
+    
+    for case in cases
+        if !isfile(case.filepath)
+            @warn "File not found: $(case.filepath). Skipping."
+            continue
+        end
+        
+        println("\nProcessing: $(case.name)")
+        
+        # --- NEW LIBRARY (JuliaMSI) with bulk processing ---
+        println("  [JuliaMSI - Bulk Processing]")
+        new_stats = try
+            @timed begin
+                # 1. Load (lazy, metadata only)
+                data = OpenMSIData(case.filepath)
+                
+                # 2. Generate ALL slices in one bulk operation
+                slice_dict = get_multiple_mz_slices(data, mz_values, tolerance)
+                
+                # 3. Process each slice (simulate quantization, saving, etc.)
+                for (mass, slice) in slice_dict
+                    # Simulate minimal processing for fair comparison
+                    # In real pipeline: TrIQ, median filter, save bitmap
+                    processed = quantize_intensity(slice, 256)[1]
+                end
+                
+                # 4. Close and cleanup
+                close(data)
+            end
+        catch e
+            @warn "New library failed: $e"
+            (time=Inf, bytes=Inf, value=nothing)
+        end
+        
+        # --- OLD LIBRARY (sequential processing) ---
+        println("  [Old Library - Sequential]")
+        old_stats = try
+            @timed begin
+                # 1. Load (everything into RAM)
+                data = LoadImzml(case.filepath)
+                
+                # 2. Generate slices one by one
+                for mass in mz_values
+                    slice = GetSlice(data, mass, tolerance)
+                    # Same processing simulation
+                    processed = IntQuant(slice)
+                end
+                
+                # 3. Old library doesn't have explicit close
+                # Data stays in RAM until GC
+            end
+        catch e
+            @warn "Old library failed: $e"
+            (time=Inf, bytes=Inf, value=nothing)
+        end
+        
+        # Memory cleanup between files (mirroring your pipeline)
+        GC.gc(true)
+        if Sys.islinux()
+            ccall(:malloc_trim, Int32, (Int32,), 0)
+        end
+        
+        push!(results, (
+            name=case.name,
+            file_size_mb=get_total_file_size_mb(case.filepath),
+            n_slices=length(mz_values),
+            julianew_time_s=new_stats.time,
+            julianew_memory_mb=new_stats.bytes/1024^2,
+            juliaold_time_s=old_stats.time,
+            juliaold_memory_mb=old_stats.bytes/1024^2,
+            speedup=old_stats.time / new_stats.time,
+            # NEW METRICS for your pipeline:
+            memory_reduction_pct=100 * (old_stats.bytes - new_stats.bytes) / old_stats.bytes,
+            slices_per_second_new=length(mz_values) / new_stats.time,
+            slices_per_second_old=length(mz_values) / old_stats.time
+        ))
+    end
+    
+    return results
+end
+
+# ===================================================================
+# Main Runner
+function run_all_benchmarks(skip_benchmark::Bool=false)
+    results_csv_path = joinpath(RESULTS_DIR, "benchmark_results.csv")
+    multi_file_walkthrough_csv_path = joinpath(RESULTS_DIR, "multi_file_walkthrough.csv")
+    real_world_pipeline_csv_path = joinpath(RESULTS_DIR, "real_world_pipeline.csv")
+
+    local all_results::DataFrame
+    local walkthrough_df::DataFrame = DataFrame() # Initialize as empty
+    local pipeline_df::DataFrame = DataFrame() # Initialize as empty
+
+    if skip_benchmark && isfile(results_csv_path) && isfile(multi_file_walkthrough_csv_path) && isfile(real_world_pipeline_csv_path)
+        @info "Skipping benchmark run. Loading results from $results_csv_path"
+        all_results = CSV.read(results_csv_path, DataFrame)
+        walkthrough_df = CSV.read(multi_file_walkthrough_csv_path, DataFrame)
+        pipeline_df = CSV.read(real_world_pipeline_csv_path, DataFrame)
+    else
+        if isempty(BENCHMARK_CASES)
+            @warn "No benchmark cases found. Please fill in the `BENCHMARK_CASES` constant in `test/benchmark.jl`."
+            return
+        end
+        
+        all_results = DataFrame()
+
+        for case in BENCHMARK_CASES
+            if !isfile(case.filepath)
+                @warn "File not found: $(case.filepath). Skipping."
+                continue
+            end
+
+            # Run for new library
+            new_results = benchmark_julianew(case)
+            push!(all_results, new_results, cols=:union)
+
+            GC.gc() # Trigger garbage collection
+            if Sys.islinux()
+                ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure julia returns the freed memory to OS
+            end
+
+            # Run for new library (Bulk)
+            new_bulk_results = benchmark_julianew_bulk(case)
+            push!(all_results, new_bulk_results, cols=:union)
+
+            GC.gc() # Trigger garbage collection
+            if Sys.islinux()
+                ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure julia returns the freed memory to OS
+            end
+
+            # Run for old library
+            old_results = benchmark_juliaold(case)
+            if old_results !== nothing
+                push!(all_results, old_results, cols=:union)
+            end
+            
+            GC.gc() # Trigger garbage collection
+            if Sys.islinux()
+                ccall(:malloc_trim, Int32, (Int32,), 0) # Ensure julia returns the freed memory to OS
+            end
+        end
+
+        # Save results to CSV (only if benchmark was actually run)
+        CSV.write(results_csv_path, all_results)
+        println("\nBenchmark results saved to $results_csv_path")
+
+        # --- Multi-File Walkthrough Benchmark ---
+        println("\n" * "="^60)
+        println("MULTI-FILE WALKTHROUGH BENCHMARK")
+        println("="^60)
+
+        walkthrough_results = benchmark_multi_file_walkthrough(BENCHMARK_CASES)
+
+        # Create a summary DataFrame
+        walkthrough_df = DataFrame(
+            name=[r.name for r in walkthrough_results],
+            file_size_mb=[r.file_size_mb for r in walkthrough_results],
+            JuliaMSI_time_s=[r.julianew_time_s for r in walkthrough_results],
+            Old_time_s=[r.juliaold_time_s for r in walkthrough_results],
+            speedup=[r.speedup for r in walkthrough_results],
+            JuliaMSI_mem_mb=[r.julianew_memory_mb for r in walkthrough_results],
+            Old_mem_mb=[r.juliaold_memory_mb for r in walkthrough_results]
+        )
+
+        println("\nMulti-file walkthrough results:")
+        show(walkthrough_df, allrows=true)
+
+        # Save to CSV
+        CSV.write(multi_file_walkthrough_csv_path, walkthrough_df)
+
+        # --- REAL-WORLD PIPELINE BENCHMARK ---
+        println("\n" * "="^60)
+        println("REAL-WORLD PIPELINE BENCHMARK")
+        println("="^60)
+
+        pipeline_results = benchmark_real_world_pipeline(BENCHMARK_CASES)
+
+        # Create a summary DataFrame
+        pipeline_df = DataFrame(
+            name=[r.name for r in pipeline_results],
+            file_size_mb=[r.file_size_mb for r in pipeline_results],
+            n_slices=[r.n_slices for r in pipeline_results],
+            JuliaMSI_time_s=[r.julianew_time_s for r in pipeline_results],
+            Old_time_s=[r.juliaold_time_s for r in pipeline_results],
+            speedup=[r.speedup for r in pipeline_results],
+            JuliaMSI_mem_mb=[r.julianew_memory_mb for r in pipeline_results],
+            Old_mem_mb=[r.juliaold_memory_mb for r in pipeline_results],
+            memory_reduction_pct=[r.memory_reduction_pct for r in pipeline_results],
+            slices_per_second_new=[r.slices_per_second_new for r in pipeline_results],
+            slices_per_second_old=[r.slices_per_second_old for r in pipeline_results]
+        )
+
+        println("\nReal-world pipeline results:")
+        show(pipeline_df, allrows=true)
+
+        # Save to CSV
+        CSV.write(real_world_pipeline_csv_path, pipeline_df)
+    end
+
+    # Generate and save plots (always happens)
+    generate_plots(all_results)
+    plot_walkthrough_results(walkthrough_df)
+    plot_real_world_pipeline_results(pipeline_df)
+end
+
+# ===================================================================
+# PLOTTING - Professional comparative visualizations
+# ===================================================================    
+function generate_plots(df::DataFrame)
+    if isempty(df)
+        println("Cannot generate plots from empty dataframe.")
+        return
+    end
+
+    unique_names = unique(df.name) # Use the new name field
+    num_files = length(unique_names)
+    
+    # Calculate positions for dodged bars
+    positions = 1:num_files # Define positions here
+    
+    # Generate A-Z labels for X-axis
+    alphabet_labels = [string(Char('A' + i - 1)) for i in 1:num_files]
+    
+    # Use a scientific color palette
+    colors = Dict(
+        "JuliaMSI" => "#2E86AB",  # Professional blue
+        "julia_mzML_imzML" => "#A23B72",  # Professional magenta
+        "JuliaMSIBulk" => "#2CA02C" # Professional green for bulk
+    )
+    
+    # Shorter labels for display
+    library_labels = Dict(
+        "JuliaMSI" => "New",
+        "julia_mzML_imzML" => "Old",
+        "JuliaMSIBulk" => "New Bulk"
+    )
+    
+    # Define specific colors for the speedup metrics for each library
+    speedup_colors_new = ["#1E88E5", "#43A047", "#FFB300", "#E53935"] # Blue, Green, Yellow, Red for JuliaMSI
+    speedup_colors_bulk = ["#0056B3", "#008000", "#FFA500", "#CC0000"] # Slightly darker/different hues for JuliaMSIBulk
+    
+    # Set modern theme
+    set_theme!(Theme(
+        fontsize = 72, # General font size (1.5x of 48)
+        font = "Helvetica", # Changed to Helvetica
+        Axis = (
+            backgroundcolor = :white,
+            spinewidth = 1.5,
+            xgridvisible = false,
+            ygridvisible = true,
+            ygridcolor = (:gray, 0.2),
+            ygridwidth = 0.5,
+            xticklabelrotation = π/8,
+            xticklabalign = (:right, :center),
+            yticklabalign = (:right, :center),
+            xlabelsize = 72, # Axis label size (1.5x of 48)
+            ylabelsize = 72, # Axis label size (1.5x of 48)
+            titlesize = 72, # Axis title size (1.5x of 48)
+            xticklabelsize = 60, # Tick label size (1.5x of 40)
+            yticklabelsize = 60, # Tick label size (1.5x of 40)
+            titlefont = :bold, # Ensured bold
+        ),
+        Legend = (
+            backgroundcolor = (:white, 0.95),
+            framecolor = :gray,
+            framewidth = 1,
+            padding = (10, 10, 5, 5),
+            labelsize = 72, # Legend label size (1.5x of 48)
+            titlesize = 72, # Legend title size (1.5x of 48)
+        )
+    ))
+
+    # ===================================================================
+    # Plot 1: Main Performance Dashboard
+    # ===================================================================
+    fig1 = Figure(size = (3600, 2400)) # Explicitly set figure size
+    
+    # Define grid layout with specific areas
+    g1 = fig1[2, 1:2] = GridLayout()
+    
+    # Total Time
+    ax1 = Axis(g1[1, 1],
+        title = "Total Time",
+        ylabel = "Seconds", # Changed to linear scale, removed "(log scale)"
+        xticks = (positions, alphabet_labels) # Use alphabetical labels for x-axis
+    )
+    
+    # Average Image Time
+    ax2 = Axis(g1[1, 2],
+        title = "Image Generation Time",
+        ylabel = "Seconds",
+        yticks = LinearTicks(5),
+        xticks = (positions, alphabet_labels) # Use alphabetical labels for x-axis
+    )
+    
+    # Loading Memory
+    ax3 = Axis(g1[2, 1],
+        title = "Loading Memory",
+        ylabel = "MB",
+        yticks = LinearTicks(5),
+        xticks = (positions, alphabet_labels) # Use alphabetical labels for x-axis
+    )
+    
+    # Image Generation Memory
+    ax4 = Axis(g1[2, 2],
+        title = "Image Generation Memory",
+        ylabel = "MB",
+        yticks = LinearTicks(5),
+        xticks = (positions, alphabet_labels) # Use alphabetical labels for x-axis
+    )
+
+    bar_width = 0.25 # Reduced bar width
+    x_offsets = [-0.3, 0, 0.3] # Offsets for 3 bars to avoid overlap
+    
+    # Arrays to store speedup factors
+    total_time_speedup_new = zeros(num_files)
+    image_time_speedup_new = zeros(num_files)
+    load_mem_speedup_new = zeros(num_files)
+    img_mem_speedup_new = zeros(num_files)
+
+    total_time_speedup_bulk = zeros(num_files)
+    image_time_speedup_bulk = zeros(num_files)
+    load_mem_speedup_bulk = zeros(num_files)
+    img_mem_speedup_bulk = zeros(num_files)
+    
+    # Initialize a list to hold rows for the summary DataFrame
+    summary_df_rows = []
+
+    for (i, name_for_df_filter) in enumerate(unique_names) # Iterate through unique names
+        # Get data for this file
+        new_data = df[(df.name .== name_for_df_filter) .& (df.library .== "JuliaMSI"), :]
+        bulk_data = df[(df.name .== name_for_df_filter) .& (df.library .== "JuliaMSIBulk"), :]
+        old_data = df[(df.name .== name_for_df_filter) .& (df.library .== "julia_mzML_imzML"), :]
+        
+        # Plot total time
+        barplot!(ax1, [i + x_offsets[1], i + x_offsets[2], i + x_offsets[3]],
+            [new_data.total_time_s[1], bulk_data.total_time_s[1], old_data.total_time_s[1]],
+            color = [colors["JuliaMSI"], colors["JuliaMSIBulk"], colors["julia_mzML_imzML"]],
+            width = bar_width
+        )
+        
+        # Plot average image time
+        barplot!(ax2, [i + x_offsets[1], i + x_offsets[2], i + x_offsets[3]],
+            [new_data.avg_image_time_s[1], bulk_data.avg_image_time_s[1], old_data.avg_image_time_s[1]],
+            color = [colors["JuliaMSI"], colors["JuliaMSIBulk"], colors["julia_mzML_imzML"]],
+            width = bar_width
+        )
+        
+        # Plot loading memory
+        barplot!(ax3, [i + x_offsets[1], i + x_offsets[2], i + x_offsets[3]],
+            [new_data.load_memory_mb[1], bulk_data.load_memory_mb[1], old_data.load_memory_mb[1]],
+            color = [colors["JuliaMSI"], colors["JuliaMSIBulk"], colors["julia_mzML_imzML"]],
+            width = bar_width
+        )
+        
+        # Plot image generation memory
+        barplot!(ax4, [i + x_offsets[1], i + x_offsets[2], i + x_offsets[3]],
+            [new_data.avg_image_memory_mb[1], bulk_data.avg_image_memory_mb[1], old_data.avg_image_memory_mb[1]],
+            color = [colors["JuliaMSI"], colors["JuliaMSIBulk"], colors["julia_mzML_imzML"]],
+            width = bar_width
+        )
+        
+        # Calculate speedup factors (higher is better for new library)
+        if nrow(new_data) == 1 && nrow(old_data) == 1
+            total_time_speedup_new[i] = old_data.total_time_s[1] / new_data.total_time_s[1]
+            image_time_speedup_new[i] = old_data.avg_image_time_s[1] / new_data.avg_image_time_s[1]
+            load_mem_speedup_new[i] = old_data.load_memory_mb[1] / new_data.load_memory_mb[1]
+            img_mem_speedup_new[i] = old_data.avg_image_memory_mb[1] / new_data.avg_image_memory_mb[1]
+        end
+        if nrow(bulk_data) == 1 && nrow(old_data) == 1
+            total_time_speedup_bulk[i] = old_data.total_time_s[1] / bulk_data.total_time_s[1]
+            image_time_speedup_bulk[i] = old_data.avg_image_time_s[1] / bulk_data.avg_image_time_s[1]
+            load_mem_speedup_bulk[i] = old_data.load_memory_mb[1] / bulk_data.load_memory_mb[1]
+            img_mem_speedup_bulk[i] = old_data.avg_image_memory_mb[1] / bulk_data.avg_image_memory_mb[1]
+        end
+
+        # Collect data for the summary table CSV
+        current_case_name = unique_names[i]
+        case_row = df[(df.name .== current_case_name) .& (df.library .== "JuliaMSI"), :] # Any library row will have file_size_mb
+        file_size = nrow(case_row) == 1 ? case_row.file_size_mb[1] : NaN
+
+        push!(summary_df_rows, Dict(
+            "DS" => unique_names[i],
+            "File Size (MB)" => file_size,
+            "JMSI-TT Speedup" => round(total_time_speedup_new[i], digits=2),
+            "JMSI-IT Speedup" => round(image_time_speedup_new[i], digits=2),
+            "JMSI-LM Speedup" => round(load_mem_speedup_new[i], digits=2),
+            "JMSI-IM Speedup" => round(img_mem_speedup_new[i], digits=2),
+            "JBulk-TT Speedup" => round(total_time_speedup_bulk[i], digits=2),
+            "JBulk-IT Speedup" => round(image_time_speedup_bulk[i], digits=2),
+            "JBulk-LM Speedup" => round(load_mem_speedup_bulk[i], digits=2),
+            "JBulk-IM Speedup" => round(img_mem_speedup_bulk[i], digits=2)
+        ))
+    end
+    
+    # Create DataFrame from collected rows
+    summary_df = DataFrame(summary_df_rows)
+    
+    # Save the summary table to CSV
+    CSV.write(joinpath(RESULTS_DIR, "benchmark_summary_table.csv"), summary_df)
+    println("\nBenchmark summary table saved to $(joinpath(RESULTS_DIR, "benchmark_summary_table.csv"))")
+
+    # Add overall title
+    Label(fig1[1, :], "MSI Library Performance Benchmark", fontsize = 72, font = :bold)
+    
+    # Add a legend
+    Legend(fig1[3, :], # Moved to row 3, spanning all columns
+        [PolyElement(color = colors["JuliaMSI"]), PolyElement(color = colors["JuliaMSIBulk"]), PolyElement(color = colors["julia_mzML_imzML"])],
+        [library_labels["JuliaMSI"], library_labels["JuliaMSIBulk"], library_labels["julia_mzML_imzML"]],
+        "Library",
+        orientation = :horizontal,
+        tellwidth = false, tellheight = true,
+        halign = :center, valign = :top, # Centered horizontally, aligned to top
+        margin = (10, 10, 10, 10)
+    )
+    
+    # Adjust layout spacing
+    colgap!(g1, 1, 30)
+    rowgap!(g1, 1, 20)
+    
+    # Ensure all rows and columns in g1 expand to fill available space
+    rowsize!(g1, 1, Auto())
+    rowsize!(g1, 2, Auto())
+    colsize!(g1, 1, Auto())
+    colsize!(g1, 2, Auto())
+    
+    # Ensure the row containing the legend expands in the top-level figure layout
+    rowsize!(fig1.layout, 3, Auto())
+
+    
+
+    # Save plots
+
+    save(joinpath(RESULTS_DIR, "performance_dashboard.png"), fig1, px_per_unit = 1)    
+    println("\nEnhanced visualizations saved to $RESULTS_DIR")
+end
+
+function plot_walkthrough_results(walkthrough_df)
+    fig = Figure(size=(3600, 2400))
+    
+    # Plot 1: Total time per file
+    ax1 = Axis(fig[1, 1], 
+               title="Time per File (Load + Generate 1 Image)",
+               ylabel="Time (seconds)",
+               xticks=(collect(1:nrow(walkthrough_df)), walkthrough_df.name),
+               xticklabelrotation=π/8)
+    
+    # Grouped bars
+    barplot!(ax1, collect(1:nrow(walkthrough_df)) .- 0.2, walkthrough_df.JuliaMSI_time_s, 
+             width=0.4, color="#2E86AB", label="JuliaMSI (New)")
+    barplot!(ax1, collect(1:nrow(walkthrough_df)) .+ 0.2, walkthrough_df.Old_time_s,
+             width=0.4, color="#A23B72", label="julia_mzML_imzML (Old)")
+    
+    # Plot 2: Speedup factor
+    ax2 = Axis(fig[1, 2],
+               title="Speedup Factor (Old/New Time)",
+               ylabel="Speedup (Higher = Better)",
+               xticks=(collect(1:nrow(walkthrough_df)), walkthrough_df.name),
+               xticklabelrotation=π/8)
+    
+    # Bars above 1 = new library is faster
+    colors = [x >= 1 ? "#2CA02C" : "#E53935" for x in walkthrough_df.speedup]
+    barplot!(ax2, collect(1:nrow(walkthrough_df)), walkthrough_df.speedup, color=colors)
+    hlines!(ax2, [1.0], color=:black, linestyle=:dash, linewidth=2)
+    
+    # Plot 3: Memory comparison
+    ax3 = Axis(fig[2, 1],
+               title="Memory Usage per File",
+               ylabel="Memory (MB)",
+               xticks=(collect(1:nrow(walkthrough_df)), walkthrough_df.name),
+               xticklabelrotation=π/8)
+    
+    barplot!(ax3, collect(1:nrow(walkthrough_df)) .- 0.2, walkthrough_df.JuliaMSI_mem_mb,
+             width=0.4, color="#2E86AB", label="JuliaMSI")
+    barplot!(ax3, collect(1:nrow(walkthrough_df)) .+ 0.2, walkthrough_df.Old_mem_mb,
+             width=0.4, color="#A23B72", label="Old Library")
+    
+    # Plot 4: Time vs File Size (scatter)
+    ax4 = Axis(fig[2, 2],
+               title="Processing Time vs File Size",
+               xlabel="File Size (MB)",
+               ylabel="Time (seconds)")
+    
+    scatter!(ax4, walkthrough_df.file_size_mb, walkthrough_df.JuliaMSI_time_s,
+             color="#2E86AB", markersize=20, label="JuliaMSI")
+    scatter!(ax4, walkthrough_df.file_size_mb, walkthrough_df.Old_time_s,
+             color="#A23B72", markersize=20, label="Old Library")
+    
+    # Add linear trend lines
+    if nrow(walkthrough_df) > 1
+        linreg_new = hcat(fill(1.0, nrow(walkthrough_df)), walkthrough_df.file_size_mb) \ walkthrough_df.JuliaMSI_time_s
+        linreg_old = hcat(fill(1.0, nrow(walkthrough_df)), walkthrough_df.file_size_mb) \ walkthrough_df.Old_time_s
+        
+        x_vals = [minimum(walkthrough_df.file_size_mb), maximum(walkthrough_df.file_size_mb)]
+        lines!(ax4, x_vals, linreg_new[1] .+ linreg_new[2] .* x_vals, color="#2E86AB", linewidth=2)
+        lines!(ax4, x_vals, linreg_old[1] .+ linreg_old[2] .* x_vals, color="#A23B72", linewidth=2)
+    end
+    
+    # Add legends
+    Legend(fig[0, :], [PolyElement(color="#2E86AB"), PolyElement(color="#A23B72")],
+           ["JuliaMSI (New)", "julia_mzML_imzML (Old)"], 
+           orientation=:horizontal, tellwidth=false, framevisible=true)
+    
+    # Adjust layout
+    rowgap!(fig.layout, 1)
+    colgap!(fig.layout, 1)
+    
+    save(joinpath(RESULTS_DIR, "multi_file_walkthrough.png"), fig, px_per_unit=1)
+    println("\nMulti-file walkthrough plot saved.")
+end
+
+function plot_real_world_pipeline_results(pipeline_df::DataFrame)
+    fig = Figure(size=(3600, 2400)) # Larger figure for more detailed plots
+
+    # Filter out rows with Inf or NaN values in critical plotting columns
+    # Create a copy to avoid modifying the original DataFrame
+    filtered_df = deepcopy(pipeline_df)
+    
+    # Filter for finite values in key metrics for plotting
+    filter!(row -> isfinite(row.JuliaMSI_time_s) && isfinite(row.Old_time_s) && 
+                   isfinite(row.speedup) && isfinite(row.JuliaMSI_mem_mb) && 
+                   isfinite(row.Old_mem_mb) && isfinite(row.memory_reduction_pct) &&
+                   isfinite(row.slices_per_second_new) && isfinite(row.slices_per_second_old), 
+            filtered_df)
+
+    if isempty(filtered_df)
+        println("No finite data points for real-world pipeline plotting after filtering. Skipping plot generation.")
+        save(joinpath(RESULTS_DIR, "real_world_pipeline.png"), fig, px_per_unit=1)
+        return
+    end
+
+    num_rows_filtered = nrow(filtered_df)
+    x_positions = collect(1:num_rows_filtered)
+    
+    # Plot 1: Total Time Comparison
+    ax1 = Axis(fig[1, 1],
+               title="Real-World Pipeline: Total Time",
+               ylabel="Time (seconds)",
+               xticks=(x_positions, filtered_df.name),
+               xticklabelrotation=π/8)
+    
+    barplot!(ax1, x_positions .- 0.2, filtered_df.JuliaMSI_time_s,
+             width=0.4, color="#2E86AB", label="JuliaMSI (New)")
+    barplot!(ax1, x_positions .+ 0.2, filtered_df.Old_time_s,
+             width=0.4, color="#A23B72", label="Old Library")
+    
+    # Plot 2: Speedup Factor
+    ax2 = Axis(fig[1, 2],
+               title="Real-World Pipeline: Speedup Factor (Old/New Time)",
+               ylabel="Speedup (Higher = Better)",
+               xticks=(x_positions, filtered_df.name),
+               xticklabelrotation=π/8)
+    
+    colors_speedup = [x >= 1 ? "#2CA02C" : "#E53935" for x in filtered_df.speedup]
+    barplot!(ax2, x_positions, filtered_df.speedup, color=colors_speedup)
+    hlines!(ax2, [1.0], color=:black, linestyle=:dash, linewidth=2)
+    
+    # Plot 3: Memory Usage Comparison
+    ax3 = Axis(fig[2, 1],
+               title="Real-World Pipeline: Memory Usage",
+               ylabel="Memory (MB)",
+               xticks=(x_positions, filtered_df.name),
+               xticklabelrotation=π/8)
+    
+    barplot!(ax3, x_positions .- 0.2, filtered_df.JuliaMSI_mem_mb,
+             width=0.4, color="#2E86AB", label="JuliaMSI")
+    barplot!(ax3, x_positions .+ 0.2, filtered_df.Old_mem_mb,
+             width=0.4, color="#A23B72", label="Old Library")
+
+    # Plot 4: Memory Reduction Percentage
+    ax4 = Axis(fig[2, 2],
+               title="Real-World Pipeline: Memory Reduction vs Old (%)",
+               ylabel="Memory Reduction (%)",
+               xticks=(x_positions, filtered_df.name),
+               xticklabelrotation=π/8)
+    
+    colors_mem_red = [x > 0 ? "#2CA02C" : "#E53935" for x in filtered_df.memory_reduction_pct]
+    barplot!(ax4, x_positions, filtered_df.memory_reduction_pct, color=colors_mem_red)
+    hlines!(ax4, [0.0], color=:black, linestyle=:dash, linewidth=2)
+
+    # Plot 5: Slices Per Second
+    ax5 = Axis(fig[3, 1],
+               title="Real-World Pipeline: Slices per Second",
+               ylabel="Slices/sec (Higher = Better)",
+               xticks=(x_positions, filtered_df.name),
+               xticklabelrotation=π/8)
+    
+    barplot!(ax5, x_positions .- 0.2, filtered_df.slices_per_second_new,
+             width=0.4, color="#2E86AB", label="JuliaMSI")
+    barplot!(ax5, x_positions .+ 0.2, filtered_df.slices_per_second_old,
+             width=0.4, color="#A23B72", label="Old Library")
+
+    # Overall Legend
+    Legend(fig[0, :], [PolyElement(color="#2E86AB"), PolyElement(color="#A23B72")],
+           ["JuliaMSI (New)", "julia_mzML_imzML (Old)"],
+           orientation=:horizontal, tellwidth=false, framevisible=true)
+    
+    # Adjust layout
+    rowgap!(fig.layout, 1)
+    colgap!(fig.layout, 1)
+    
+    save(joinpath(RESULTS_DIR, "real_world_pipeline.png"), fig, px_per_unit=1)
+    println("\nReal-world pipeline plot saved.")
+end
+
+# --- Execute ---
+mkpath(RESULTS_DIR)
+run_all_benchmarks(true) # true: skip processing files # false: create new matrix

test/benchmark_v3.jl

@@ -0,0 +1,215 @@
+# test/benchmark_v3.jl
+
+# ===================================================================
+# High-Precision Performance Benchmark Suite (v3)
+# ===================================================================
+# This script integrates the robust statistical sampling of `BenchmarkTools`
+# with the visual comparative mechanics against the legacy library.
+# It captures the paradigm shift from "Time per slice" to "Pipeline Throughput".
+# ===================================================================
+
+using Pkg
+Pkg.activate(joinpath(@__DIR__, ".."))
+
+using BenchmarkTools
+using DataFrames
+using CSV
+using CairoMakie
+using Statistics
+using MSI_src
+using julia_mzML_imzML
+
+struct BenchmarkCase
+    filepath::String
+    mz_value::Float64
+    mz_tolerance::Float64
+    name::String
+end
+
+const RESULTS_DIR = joinpath(@__DIR__, "results")
+
+# List to benchmark
+const BENCHMARK_CASES = [
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/Chilli/ltpmsi-chilli.imzML", 420.0, 0.1, "Chilli Pepper"),
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_DESI/40TopL,10TopR,30BottomL,20BottomR/40TopL,10TopR,30BottomL,20BottomR-centroid.imzML", 885.5, 0.1, "Colon Cancer Human"),
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML", 716.053, 0.1, "Mouse Urinary Bladder"),
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/Leafs/CE1_Leaf_R3.imzML", 306.1, 0.1, "Leaf"),
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/Liv2_imzML_TIMSConvert-selected/Liv2.imzML", 796.18, 0.1, "Liver Cut"),
+    BenchmarkCase("/home/pixel/Documents/Cinvestav_2025/Analisis/salida_Tims/Stomach_DHB.imzML", 804.3, 0.1, "Mouse Stomach 4GB")
+]
+
+function get_total_file_size_mb(filepath::String)
+    imzml_size = isfile(filepath) ? filesize(filepath) : 0
+    ibd_path = replace(filepath, r"\.(imzML|imzml)$" => ".ibd")
+    ibd_size = isfile(ibd_path) ? filesize(ibd_path) : 0
+    return round((imzml_size + ibd_size) / 1024^2, digits=2)
+end
+
+function flush_memory()
+    GC.gc(true)
+    if Sys.islinux()
+        # Force the OS to reclaim memory from the glibc allocator
+        ccall(:malloc_trim, Int32, (Int32,), 0)
+    end
+end
+
+function run_v3_benchmarks()
+    mkpath(RESULTS_DIR)
+    results = DataFrame()
+
+    println("="^60)
+    println("STARTING ENTERPRISE BENCHMARK SUITE (v3)")
+    println("="^60)
+
+    for case in BENCHMARK_CASES
+        if !isfile(case.filepath)
+            @warn "File not found: $(case.filepath). Skipping."
+            continue
+        end
+
+        println("\n--- Target: $(case.name) ---")
+        file_size = get_total_file_size_mb(case.filepath)
+
+        # ------------------------------------------------------------
+        # 1. JuliaMSI (New Architecture)
+        # ------------------------------------------------------------
+        println("[JuliaMSI - New Engine]")
+        flush_memory()
+        
+        # Load Phase (Metadata Only / Memory Mapping)
+        load_stats_new = @timed OpenMSIData(case.filepath)
+        msi_data = load_stats_new.value
+        load_time_new_s = load_stats_new.time
+        mem_load_new_mb = load_stats_new.bytes / 1024^2
+        
+        # Slicing Phase (High Precision)
+        b_slice_new = @benchmark get_mz_slice($msi_data, $(case.mz_value), $(case.mz_tolerance)) samples=10 seconds=5
+        mean_time_new_s = mean(b_slice_new.times) / 1e9
+        
+        # Calculate Amortized Throughput (10 slices)
+        # Includes the 'Loading Wall' penalty
+        total_time_10_new = load_time_new_s + (10 * mean_time_new_s)
+        amortized_sps_new = 10.0 / total_time_10_new
+        
+        close(msi_data)
+        flush_memory()
+
+        # ------------------------------------------------------------
+        # 2. julia_mzML_imzML (Legacy Architecture)
+        # ------------------------------------------------------------
+        println("[julia_mzML_imzML - Legacy]")
+        
+        load_stats_old = try
+            @timed LoadImzml(case.filepath)
+        catch e
+            @warn "Legacy load failed: $e"
+            (time=Inf, bytes=Inf, value=nothing)
+        end
+        
+        old_data = load_stats_old.value
+        mem_load_old_mb = load_stats_old.bytes / 1024^2
+        
+        load_time_old_s = Inf
+        mean_time_old_s = Inf
+        amortized_sps_old = 0.0
+        
+        if old_data !== nothing
+            b_slice_old = try
+                # Legacy can be extremely slow, constrain it heavily
+                @benchmark GetSlice($old_data, $(case.mz_value), $(case.mz_tolerance)) samples=3 seconds=10
+            catch e
+                @warn "Legacy slice failed: $e"
+                nothing
+            end
+            
+            if b_slice_old !== nothing
+                mean_time_old_s = mean(b_slice_old.times) / 1e9
+                
+                # Accurately reflect the massive load time in the throughput
+                load_time_old_s = load_stats_old.time
+                total_time_10_old = load_time_old_s + (10 * mean_time_old_s)
+                amortized_sps_old = 10.0 / total_time_10_old
+            end
+        end
+        
+        flush_memory()
+
+        # ------------------------------------------------------------
+        # Record
+        # ------------------------------------------------------------
+        push!(results, (
+            Dataset = case.name,
+            FileSize_MB = file_size,
+            
+            # Legacy Metrics
+            Legacy_LoadMem_MB = mem_load_old_mb,
+            Legacy_LoadTime_s = load_stats_old.time,
+            Legacy_Amortized_SPS_10 = amortized_sps_old,
+            
+            # New Metrics
+            New_LoadMem_MB = mem_load_new_mb,
+            New_LoadTime_s = load_time_new_s,
+            New_Amortized_SPS_10 = amortized_sps_new,
+            
+            # Competitive Deltas
+            RAM_Reduction_Pct = isfinite(mem_load_old_mb) ? ((mem_load_old_mb - mem_load_new_mb) / mem_load_old_mb)*100 : NaN,
+            UX_Speedup_Factor = isfinite(amortized_sps_old) ? (amortized_sps_new / amortized_sps_old) : NaN
+        ))
+        
+        println("  > RAM Reduction: $(round(results[end, :RAM_Reduction_Pct], digits=2))%")
+        println("  > Amortized Throughput (10 Slices): $(round(amortized_sps_old, digits=2)) -> $(round(amortized_sps_new, digits=2)) slices/sec")
+    end
+
+    csv_path = joinpath(RESULTS_DIR, "v3_enterprise_benchmarks.csv")
+    CSV.write(csv_path, results)
+    println("\nData saved to $csv_path")
+    
+    plot_v3_results(results)
+    return results
+end
+
+function plot_v3_results(df::DataFrame)
+    if isempty(df) return end
+
+    fig = Figure(size=(1800, 1200), fontsize=24)
+    x_pos = 1:nrow(df)
+    labels = df.Dataset
+
+    # ---------- Plot 1: The RAM Revolution ----------
+    ax1 = Axis(fig[1, 1], 
+               title="Initial RAM Cost (Loading & Mmap)", 
+               ylabel="Memory (MB)",
+               xticks=(x_pos, labels), xticklabelrotation=π/8)
+    
+    barplot!(ax1, x_pos .- 0.2, df.New_LoadMem_MB, color="#10b981", width=0.4, label="JuliaMSI (Mmap Lazy)")
+    barplot!(ax1, x_pos .+ 0.2, df.Legacy_LoadMem_MB, color="#ef4444", width=0.4, label="Legacy (Dense Matrix)")
+    axislegend(ax1, position=:lt)
+
+    # ---------- Plot 2: Throughput Leap ----------
+    ax2 = Axis(fig[1, 2], 
+               title="Amortized Throughput (10 Slices - Higher is Better)", 
+               ylabel="Slices / Second (Inc. Load Time)",
+               xticks=(x_pos, labels), xticklabelrotation=π/8)
+               
+    barplot!(ax2, x_pos .- 0.2, df.New_Amortized_SPS_10, color="#10b981", width=0.4, label="JuliaMSI")
+    barplot!(ax2, x_pos .+ 0.2, df.Legacy_Amortized_SPS_10, color="#ef4444", width=0.4, label="Legacy")
+    axislegend(ax2, position=:lt)
+
+    # ---------- Plot 3: The Scaling Wall (File Size vs Throughput) ----------
+    ax3 = Axis(fig[2, 1:2], 
+               title="Performance Scaling: UX Throughput vs Dataset Size", 
+               xlabel="Dataset File Size (MB)", 
+               ylabel="Amortized Throughput (Slices/sec)")
+               
+    scatterlines!(ax3, df.FileSize_MB, df.New_Amortized_SPS_10, color="#10b981", markersize=15, linewidth=4, label="JuliaMSI Engine")
+    scatterlines!(ax3, df.FileSize_MB, df.Legacy_Amortized_SPS_10, color="#ef4444", markersize=15, linewidth=4, label="Legacy Engine")
+    axislegend(ax3, position=:rt)
+
+    save(joinpath(RESULTS_DIR, "v3_enterprise_dashboard.png"), fig)
+    println("\nDashboards saved successfully.")
+end
+
+# Ensure we process if run as the main script
+if abspath(PROGRAM_FILE) == @__FILE__
+    run_v3_benchmarks()
+end

test/new_benchmark_mmap.jl

@@ -0,0 +1,48 @@
+using BenchmarkTools
+using MSI_src
+using Statistics
+using DataFrames
+
+# ===================================================================
+# HIGH-PRECISION COMPARATIVE SUITE
+# ===================================================================
+
+function run_advanced_benchmark(path, mz, tol)
+    println("\n" * "="^40)
+    println("TARGET: $(basename(path))")
+    println("="^40)
+
+    # 1. NEW LIBRARY: Metadata Load (The "Control Tower" startup)
+    # This measures how fast the Mmap and Cache system works
+    t_load_new = @belapsed OpenMSIData($path)
+    
+    # 2. NEW LIBRARY: Slice Generation (The "Streaming" speed)
+    msi_new = OpenMSIData(path)
+    # We use @benchmark to get a distribution (min, mean, max)
+    b_slice_new = @benchmark get_mz_slice($msi_new, $mz, $tol)
+
+    # --- Metrics Table ---
+    results = DataFrame(
+        Metric = ["Metadata Load", "Slice Gen (Min)", "Slice Gen (Mean)", "Allocations"],
+        JuliaMSI = [
+            "$(round(t_load_new * 1000, digits=2)) ms",
+            "$(round(minimum(b_slice_new.times)/1e6, digits=2)) ms",
+            "$(round(mean(b_slice_new.times)/1e6, digits=2)) ms",
+            "$(b_slice_new.allocs) allocs"
+        ]
+    )
+    
+    println(results)
+    
+    # --- The "Throughput" Test ---
+    # How many slices per second can we handle?
+    throughput_new = 1.0 / mean(b_slice_new.times/1e9)
+    println("\nThroughput: $(round(throughput_new, digits=1)) slices/sec")
+    
+    return results
+end
+
+# Example Run
+@time run_advanced_benchmark("/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML", 716.053, 0.1)
+
+# For multithread: julia --threads auto --project=. test/new_benchmark_mmap.jl

test/run_descriptive_preprocessing.jl

@@ -2,221 +2,322 @@ using CairoMakie
 using Statistics
 using Colors
 
-function create_msi_parameter_plot()
+"""
-    # Create the figure with subplots for each preprocessing step
+Plots the effect of baseline correction, showing how different numbers of SNIP
-    fig = Figure(size=(1600, 1200), fontsize=12)
+iterations affect the estimated baseline.
+"""
+function plot_baseline_correction_details()
+    fig = Figure(size=(1200, 700), fontsize=14)
+    ax = Axis(fig[1, 1], title="Detailed View: Baseline Correction (SNIP)", xlabel="m/z", ylabel="Intensity")
 
-    # Define the preprocessing steps and their parameters
+    mz_range = range(200, 1000, length=1500)
-    steps = [
+    true_signal = 1.2 .* exp.(-0.001 .* (mz_range .- 450).^2) .+ 0.8 .* exp.(-0.002 .* (mz_range .- 750).^2)
-        ("BaselineCorrection", ["iterations: 10", "method: SNIP", "window: 3.0"]),
+    baseline_comp = 0.15 .+ 0.1 .* sin.(mz_range ./ 60) .+ 0.0001 .* mz_range
-        ("Calibration", ["fit_order: 2", "method: internal_standards", "ppm_tolerance: 20.0"]),
+    noise = 0.04 .* randn(length(mz_range))
-        ("Normalization", ["method: rms"]),
+    raw_signal = true_signal .+ baseline_comp .+ noise
-        ("PeakAlignment", ["max_shift_ppm: 50.0", "method: linear", "tolerance: 20.0 ppm"]),
-        ("PeakBinningParams", ["max_bin_width_ppm: 60.0", "method: adaptive", "min_peak_per_bin: 3"]),
-        ("PeakPicking", ["half_window: 5", "method: centroid", "snr_threshold: 15.0"]),
-        ("PeakSelection", ["max_fwhm_ppm: 95.39", "min_fwhm_ppm: 19.08", "min_snr: 3.0"]),
-        ("Smoothing", ["method: savitzky_golay", "order: 3", "window: 5"])
-    ]
 
-    # Create a grid of subplots
+    # Simplified SNIP simulation for visualization
-    g = fig[1, 1] = GridLayout()
+    function simple_snip(y, iterations)
-    
+        b = copy(y)
-    # Plot each preprocessing step
+        for _ in 1:iterations
-    for (idx, (step_name, params)) in enumerate(steps)
+            for i in 2:length(b)-1
-        row, col = fldmod1(idx, 2)
+                b[i] = min(b[i], 0.5 * (b[i-1] + b[i+1]))
-        ax = Axis(g[row, col], title=step_name, titlesize=14)
-        
-        # Generate simulated m/z values and intensities
-        mz_range = range(100, 1000, length=500)
-        
-        if step_name == "BaselineCorrection"
-            # Simulate spectrum with baseline
-            true_signal = 0.5 .* exp.(-0.001 .* (mz_range .- 400).^2) .+
-                          0.3 .* exp.(-0.002 .* (mz_range .- 600).^2)
-            baseline = 0.1 .+ 0.05 .* sin.(mz_range ./ 50)
-            noisy_signal = true_signal .+ baseline .+ 0.02 .* randn(length(mz_range))
-            corrected = noisy_signal .- baseline
-            
-            lines!(ax, mz_range, noisy_signal, color=:blue, linewidth=2, label="Raw")
-            lines!(ax, mz_range, baseline, color=:red, linewidth=2, linestyle=:dash, label="Baseline")
-            lines!(ax, mz_range, corrected, color=:green, linewidth=2, label="Corrected")
-            
-        elseif step_name == "Calibration"
-            # Simulate calibration shift
-            reference_peaks = [200, 400, 600, 800]
-            measured_peaks = reference_peaks .+ 2.0 .* randn(length(reference_peaks))
-            
-            scatter!(ax, reference_peaks, fill(0.5, length(reference_peaks)), 
-                    color=:red, markersize=15, label="Reference")
-            scatter!(ax, measured_peaks, fill(0.3, length(measured_peaks)), 
-                    color=:blue, markersize=10, label="Measured")
-            
-            # Add calibration lines
-            for i in 1:length(reference_peaks)
-                lines!(ax, [measured_peaks[i], reference_peaks[i]], [0.3, 0.5], 
-                      color=:black, linewidth=1, linestyle=:dash)
-            end
-            
-        elseif step_name == "Normalization"
-            # Simulate normalization effect
-            spectra = [
-                0.8 .* exp.(-0.001 .* (mz_range .- 300).^2) .+ 0.2 .* randn(length(mz_range)),
-                1.2 .* exp.(-0.001 .* (mz_range .- 300).^2) .+ 0.2 .* randn(length(mz_range)),
-                0.9 .* exp.(-0.001 .* (mz_range .- 300).^2) .+ 0.2 .* randn(length(mz_range))
-            ]
-            
-            normalized_spectra = [spec ./ std(spec) for spec in spectra]
-            
-            for (i, spec) in enumerate(spectra)
-                lines!(ax, mz_range, spec .+ i*0.3, color=RGBA(1, 0, 0, 0.6), linewidth=2, 
-                      label=i==1 ? "Before Norm" : "")
-            end
-            for (i, spec) in enumerate(normalized_spectra)
-                lines!(ax, mz_range, spec .+ i*0.3, color=RGBA(0, 0, 1, 0.6), linewidth=2, 
-                      label=i==1 ? "After Norm" : "")
-            end
-            
-        elseif step_name == "PeakAlignment"
-            # Simulate peak alignment
-            base_peaks = [300, 500, 700]
-            shifts = [-15, 5, 10]
-            
-            for (i, shift) in enumerate(shifts)
-                shifted_peaks = base_peaks .+ shift
-                aligned_peaks = base_peaks
-                
-                scatter!(ax, shifted_peaks, fill(i, length(shifted_peaks)), 
-                        color=:red, markersize=12, label=i==1 ? "Before Align" : "")
-                scatter!(ax, aligned_peaks, fill(i+0.3, length(aligned_peaks)), 
-                        color=:green, markersize=12, label=i==1 ? "After Align" : "")
-                
-                # Show alignment lines
-                for j in 1:length(base_peaks)
-                    lines!(ax, [shifted_peaks[j], aligned_peaks[j]], [i, i+0.3], 
-                          color=:black, linewidth=1, linestyle=:dash)
             end
         end
-            
+        return b
-        elseif step_name == "PeakBinningParams"
-            # Simulate peak binning with centroids
-            raw_peaks_mz = 100:25:900
-            raw_peaks_intensity = rand(length(raw_peaks_mz))
-            
-            # Create binned peaks (wider bins)
-            bin_centers = 150:60:850
-            bin_intensities = [sum(raw_peaks_intensity[abs.(raw_peaks_mz .- center) .< 30]) 
-                              for center in bin_centers] .* 0.8
-            
-            # Profile mode (continuous)
-            profile_signal = zeros(length(mz_range))
-            for (mz, int) in zip(raw_peaks_mz, raw_peaks_intensity)
-                profile_signal .+= int .* exp.(-0.001 .* (mz_range .- mz).^2)
     end
     
-            lines!(ax, mz_range, profile_signal, color=:blue, linewidth=2, label="Profile")
+    baseline_iter_20 = simple_snip(raw_signal, 20)
-            barplot!(ax, bin_centers, bin_intensities, color=RGBA(1, 0, 0, 0.7), 
+    baseline_iter_200 = simple_snip(raw_signal, 200)
-                    width=50, label="Binned Centroids")
+    corrected_signal = raw_signal .- baseline_iter_200
 
-        elseif step_name == "PeakPicking"
+    lines!(ax, mz_range, raw_signal, color=(:grey, 0.6), label="Raw Signal")
-            # Simulate peak picking from profile to centroids
+    lines!(ax, mz_range, corrected_signal, color=:green, linewidth=2.5, label="Corrected Signal")
-            profile_signal = 0.6 .* exp.(-0.0005 .* (mz_range .- 400).^2) .+
+    l1 = lines!(ax, mz_range, baseline_iter_20, color=(:red, 0.7), linestyle=:dash, linewidth=2, label="Baseline (iterations: 20)")
-                             0.4 .* exp.(-0.0008 .* (mz_range .- 650).^2) .+
+    l2 = lines!(ax, mz_range, baseline_iter_200, color=:red, linewidth=2.5, label="Baseline (iterations: 200)")
-                             0.1 .* randn(length(mz_range))
 
-            # Simulate picked peaks (centroids)
+    text!(ax, "Parameter: `iterations`\nMore iterations result in a more aggressive baseline that follows the signal floor more closely.", 
-            peak_positions = [380, 405, 640, 660]
+          position=Point2f(0.05, 0.95), space=:relative, align=(:left, :top), fontsize=12)
-            peak_intensities = [0.5, 0.6, 0.35, 0.4]
+    axislegend(ax, position=:rt)
-            
+    save("descriptive_plot_baseline_correction.png", fig)
-            lines!(ax, mz_range, profile_signal, color=:blue, linewidth=2, label="Profile Spectrum")
+    println("Saved: descriptive_plot_baseline_correction.png")
-            scatter!(ax, peak_positions, peak_intensities, color=:red, markersize=20, 
-                    label="Picked Centroids", strokewidth=2)
-            
-        elseif step_name == "PeakSelection"
-            # Simulate peak selection based on criteria
-            all_peaks_mz = 200:50:800
-            all_peaks_fwhm = rand(length(all_peaks_mz)) .* 100 .+ 10
-            all_peaks_snr = rand(length(all_peaks_mz)) .* 10
-            
-            # Selection criteria
-            selected = (all_peaks_fwhm .>= 19.08) .& (all_peaks_fwhm .<= 95.39) .& (all_peaks_snr .>= 3.0)
-            
-            scatter!(ax, all_peaks_mz[.!selected], all_peaks_fwhm[.!selected], 
-                    color=:red, markersize=15, label="Rejected")
-            scatter!(ax, all_peaks_mz[selected], all_peaks_fwhm[selected], 
-                    color=:green, markersize=15, label="Selected")
-            
-            # Add selection criteria lines
-            hlines!(ax, [19.08, 95.39], color=:black, linestyle=:dash, linewidth=2)
-            text!(ax, 850, 50; text="FWHM bounds", color=:black, fontsize=10)
-            
-        elseif step_name == "Smoothing"
-            # Simulate smoothing effect
-            true_signal = 0.7 .* exp.(-0.001 .* (mz_range .- 450).^2) .+
-                          0.5 .* exp.(-0.0008 .* (mz_range .- 650).^2)
-            noisy_signal = true_signal .+ 0.1 .* randn(length(mz_range))
-            
-            # Simple smoothing simulation
-            smoothed = similar(noisy_signal)
-            window = 5
-            for i in 1:length(noisy_signal)
-                start_idx = max(1, i - window ÷ 2)
-                end_idx = min(length(noisy_signal), i + window ÷ 2)
-                smoothed[i] = mean(noisy_signal[start_idx:end_idx])
-            end
-            
-            lines!(ax, mz_range, noisy_signal, color=:red, linewidth=1, label="Noisy")
-            lines!(ax, mz_range, smoothed, color=:blue, linewidth=2, label="Smoothed")
-            lines!(ax, mz_range, true_signal, color=:green, linewidth=1, linestyle=:dash, label="True")
-        end
-        
-        # Add parameter text using a more reliable approach
-        param_text = join(params, "\n")
-        
-        text!(ax, param_text, position=Point2f(0.05, 0.95), 
-            space=:relative, align=(:left, :top), color=:black, 
-            fontsize=10, font=:regular)
-        
-        # Add legend for selected plots
-        if idx <= 4
-            axislegend(ax, position=:rt, framevisible=true, backgroundcolor=RGBA(1,1,1,0.8))
-        end
-        
-        # Customize axes
-        ax.xlabel = "m/z"
-        ax.ylabel = idx in [1,3,5,7] ? "Intensity" : ""
-        ax.xgridvisible = false
-        ax.ygridvisible = false
-    end
-    
-    # Add overall title
-    Label(fig[0, :], "MSI Preprocessing Pipeline Parameters and Simulations", 
-          fontsize=18, font=:bold, padding=(0, 0, 10, 0))
-    
-    # Add explanation
-    explanation = """
-    Simulation of MSI preprocessing parameters showing:
-    • Blue lines: Profile/continuous spectra
-    • Red bars/points: Centroid data  
-    • Dashed lines: Reference/true signals
-    • Green: Processed/corrected data
-    • Each subplot demonstrates key parameters for the preprocessing step
-    """
-    
-    Label(fig[2, :], explanation, fontsize=12, tellwidth=false, padding=(10, 10, 10, 10))
-    
-    # Adjust layout
-    colgap!(g, 20)
-    rowgap!(g, 20)
-    
-    fig
 end
 
-# Create and display the plot
+"""
-fig = create_msi_parameter_plot()
+Plots the effect of smoothing, comparing different window sizes.
+"""
+function plot_smoothing_details()
+    fig = Figure(size=(1200, 700), fontsize=14)
+    ax = Axis(fig[1, 1], title="Detailed View: Smoothing (Savitzky-Golay)", xlabel="m/z", ylabel="Intensity")
 
-# Save the plot
+    mz_range = range(400, 700, length=1000)
-save("msi_preprocessing_parameters.png", fig)
+    true_signal = 0.8 .* exp.(-0.002 .* (mz_range .- 500).^2) .+ 0.6 .* exp.(-0.001 .* (mz_range .- 600).^2)
-println("Plot saved as 'msi_preprocessing_parameters.png'")
+    noisy_signal = true_signal .+ 0.1 .* randn(length(mz_range))
+
+    function simple_moving_average(y, window)
+        smoothed = similar(y)
+        for i in 1:length(y)
+            start_idx = max(1, i - window ÷ 2)
+            end_idx = min(length(y), i + window ÷ 2)
+            smoothed[i] = mean(y[start_idx:end_idx])
+        end
+        return smoothed
+    end
+    
+    smoothed_small_window = simple_moving_average(noisy_signal, 5)
+    smoothed_large_window = simple_moving_average(noisy_signal, 21)
+
+    lines!(ax, mz_range, noisy_signal, color=(:red, 0.4), label="Noisy Signal")
+    lines!(ax, mz_range, true_signal, color=:black, linestyle=:dash, linewidth=2, label="True Signal")
+    lines!(ax, mz_range, smoothed_small_window, color=:blue, linewidth=2, label="Smoothed (window: 5)")
+    lines!(ax, mz_range, smoothed_large_window, color=:purple, linewidth=2, label="Smoothed (window: 21)")
+    
+    text!(ax, "Parameter: `window`\nA larger window increases smoothing but may broaden peaks.", 
+          position=Point2f(0.05, 0.95), space=:relative, align=(:left, :top), fontsize=12)
+    axislegend(ax, position=:rt)
+    save("descriptive_plot_smoothing.png", fig)
+    println("Saved: descriptive_plot_smoothing.png")
+end
+
+"""
+Visualizes the peak picking process for profile-mode data, illustrating the
+effects of SNR threshold and peak prominence.
+"""
+function plot_peak_picking_profile_details()
+    fig = Figure(size=(1200, 700), fontsize=14)
+    ax = Axis(fig[1, 1], title="Detailed View: Peak Picking (Profile Mode)", xlabel="m/z", ylabel="Intensity")
+
+    mz = 1:200
+    base_signal = 10 .* exp.(-((mz .- 50).^2) ./ (2*3^2)) .+ 7 .* exp.(-((mz .- 120).^2) ./ (2*5^2)) .+ 2
+    small_peak_signal = zeros(200)
+    for i in 80:90
+        small_peak_signal[i] = 3 * exp(-((i - 85)^2) / 2.0)
+    end
+    noise = 0.5 .* randn(200)
+    intensity = base_signal .+ small_peak_signal .+ noise
+    
+    noise_level = median(abs.(intensity .- median(intensity))) * 1.4826 # MAD
+    snr_threshold_val = 3.0
+    intensity_threshold = noise_level * snr_threshold_val
+
+    picked_peaks_mz = [50, 85, 120]
+    picked_peaks_intensity = intensity[picked_peaks_mz]
+    
+    hlines!(ax, [noise_level], color=:gray, linestyle=:dot, label="Est. Noise Level")
+    hlines!(ax, [intensity_threshold], color=:orange, linestyle=:dash, label="SNR Threshold (snr_threshold = 3.0)")
+    lines!(ax, mz, intensity, color=:blue, label="Profile Spectrum")
+    scatter!(ax, picked_peaks_mz, picked_peaks_intensity, color=:green, markersize=15, strokewidth=2, label="Peaks passing SNR")
+    scatter!(ax, [25], [intensity[25]], color=:red, marker=:x, markersize=15, label="Local max below SNR")
+
+    # Illustrate prominence
+    arrows!(ax, [120, 120], [intensity[135], intensity[120]], [0, 0], [intensity[120]-intensity[135], 0], color=:purple)
+    text!(ax, 125, (intensity[120]+intensity[135])/2, text="Prominence", color=:purple)
+
+    axislegend(ax, position=:rt)
+    save("descriptive_plot_peakpicking_profile.png", fig)
+    println("Saved: descriptive_plot_peakpicking_profile.png")
+end
+
+"""
+Visualizes peak picking (filtering) for centroid-mode data based on an SNR
+(intensity) threshold.
+"""
+function plot_peak_picking_centroid_details()
+    fig = Figure(size=(1200, 700), fontsize=14)
+    ax = Axis(fig[1, 1], title="Detailed View: Peak Picking (Centroid Mode)", xlabel="m/z", ylabel="Intensity")
+
+    mz = [100, 150, 200, 250, 300, 350, 400]
+    intensity = [10, 5, 25, 8, 3, 18, 12]
+    snr_threshold_val = 10.0
+
+    stem!(ax, mz, intensity, color=:gray, label="Input Centroids")
+    
+    selected_mask = intensity .>= snr_threshold_val
+    stem!(ax, mz[selected_mask], intensity[selected_mask], color=:green, trunkwidth=3, label="Selected Peaks (intensity >= 10)")
+    stem!(ax, mz[.!selected_mask], intensity[.!selected_mask], color=:red, trunkwidth=3, label="Rejected Peaks (intensity < 10)")
+    
+    hlines!(ax, [snr_threshold_val], color=:orange, linestyle=:dash, label="Intensity Threshold (snr_threshold)")
+
+    text!(ax, "Parameter: `snr_threshold`\nIn centroid mode, this acts as a direct intensity filter.", 
+          position=Point2f(0.05, 0.95), space=:relative, align=(:left, :top), fontsize=12)
+    axislegend(ax, position=:rt)
+    save("descriptive_plot_peakpicking_centroid.png", fig)
+    println("Saved: descriptive_plot_peakpicking_centroid.png")
+end
+
+"""
+Plots the effect of different normalization methods on a set of spectra.
+"""
+function plot_normalization_details()
+    fig = Figure(size=(1200, 700), fontsize=14)
+    
+    mz = range(300, 400, length=500)
+    spec1 = 1.5 .* exp.(-((mz .- 350).^2) ./ 50)
+    spec2 = 0.8 .* exp.(-((mz .- 350).^2) ./ 50)
+    
+    ax1 = Axis(fig[1, 1], title="Before Normalization", ylabel="Absolute Intensity")
+    lines!(ax1, mz, spec1, label="Spectrum A (High TIC)")
+    lines!(ax1, mz, spec2, label="Spectrum B (Low TIC)")
+    axislegend(ax1)
+
+    ax2 = Axis(fig[1, 2], title="After Normalization (TIC)", ylabel="Relative Intensity")
+    lines!(ax2, mz, spec1 ./ sum(spec1), label="Spectrum A (Normalized)")
+    lines!(ax2, mz, spec2 ./ sum(spec2), label="Spectrum B (Normalized)")
+
+    text!(ax2, "Effect: Spectra are scaled to have the same total area, making their intensities comparable.",
+          position=Point2f(0.05, 0.95), space=:relative, align=(:left, :top), fontsize=12, justification=:left)
+    
+    save("descriptive_plot_normalization.png", fig)
+    println("Saved: descriptive_plot_normalization.png")
+end
+
+"""
+Illustrates mass calibration, showing how a calibration curve corrects measured
+m/z values based on reference peaks.
+"""
+function plot_calibration_details()
+    fig = Figure(size=(1200, 700), fontsize=14)
+    ax = Axis(fig[1, 1], title="Detailed View: Calibration", xlabel="Measured m/z", ylabel="m/z Error (Measured - Reference)")
+
+    ref_mz = [200, 400, 600, 800, 1000]
+    measured_mz = ref_mz .+ [0.1, 0.15, 0.2, 0.25, 0.3] .+ 0.02 .* randn(5)
+    errors = measured_mz .- ref_mz
+
+    # Fit a linear model to the error
+    A = [ones(5) measured_mz]
+    coeffs = A \ errors
+    correction_func(m) = m - (coeffs[1] .+ coeffs[2] .* m)
+    
+    fit_line = coeffs[1] .+ coeffs[2] .* measured_mz
+    
+    scatter!(ax, measured_mz, errors, color=:red, markersize=15, label="Measured Error")
+    lines!(ax, measured_mz, fit_line, color=:blue, label="Calibration Curve (fit_order=1)")
+
+    text!(ax, "Parameter: `fit_order`\nA curve is fit to the error of known reference peaks.\nThis curve is then used to correct all m/z values.", 
+          position=Point2f(0.05, 0.95), space=:relative, align=(:left, :top), fontsize=12)
+    axislegend(ax, position=:rb)
+    save("descriptive_plot_calibration.png", fig)
+    println("Saved: descriptive_plot_calibration.png")
+end
+
+"""
+Visualizes peak alignment by showing multiple spectra with misaligned peaks
+before and after the alignment process.
+"""
+function plot_alignment_details()
+    fig = Figure(size=(1600, 600), fontsize=14)
+    ax1 = Axis(fig[1, 1], title="Before Alignment", xlabel="m/z", yticklabelsvisible=false, ygridvisible=false)
+    ax2 = Axis(fig[1, 2], title="After Alignment", xlabel="m/z", yticklabelsvisible=false, ygridvisible=false)
+
+    mz = range(490, 510, length=1000)
+    shifts = [-0.5, 0.0, 0.8]
+    colors = [:blue, :green, :purple]
+    
+    for (i, shift) in enumerate(shifts)
+        peak_center = 500 + shift
+        spectrum = exp.(-((mz .- peak_center).^2) ./ 0.1)
+        lines!(ax1, mz, spectrum .+ i, color=colors[i])
+        vlines!(ax1, [peak_center], color=(colors[i], 0.5), linestyle=:dash)
+
+        # After alignment, all peaks are at 500
+        aligned_spectrum = exp.(-((mz .- 500).^2) ./ 0.1)
+        lines!(ax2, mz, aligned_spectrum .+ i, color=colors[i])
+    end
+    vlines!(ax2, [500], color=:red, linestyle=:dash, label="Reference m/z")
+    axislegend(ax2)
+
+    save("descriptive_plot_alignment.png", fig)
+    println("Saved: descriptive_plot_alignment.png")
+end
+
+"""
+Illustrates peak selection by filtering a population of peaks based on
+FWHM and SNR criteria.
+"""
+function plot_peak_selection_details()
+    fig = Figure(size=(1200, 700), fontsize=14)
+    ax = Axis(fig[1, 1], title="Detailed View: Peak Selection", xlabel="FWHM (ppm)", ylabel="Signal-to-Noise Ratio (SNR)")
+
+    n_peaks = 100
+    fwhm = rand(n_peaks) .* 150
+    snr = rand(n_peaks) .* 20
+    
+    min_fwhm_ppm = 20.0
+    max_fwhm_ppm = 100.0
+    min_snr = 5.0
+
+    selected_mask = (fwhm .>= min_fwhm_ppm) .& (fwhm .<= max_fwhm_ppm) .& (snr .>= min_snr)
+
+    scatter!(ax, fwhm[.!selected_mask], snr[.!selected_mask], color=(:red, 0.5), label="Rejected Peaks")
+    scatter!(ax, fwhm[selected_mask], snr[selected_mask], color=:green, label="Selected Peaks")
+
+    vlines!(ax, [min_fwhm_ppm, max_fwhm_ppm], color=:blue, linestyle=:dash, label="FWHM bounds")
+    hlines!(ax, [min_snr], color=:orange, linestyle=:dash, label="SNR bound")
+    
+    poly!(ax, BBox(min_fwhm_ppm, max_fwhm_ppm, min_snr, 22), color=(:green, 0.1))
+    text!(ax, "Selection Region", position=(60, 12), color=:green, fontsize=14)
+
+    axislegend(ax)
+    save("descriptive_plot_peak_selection.png", fig)
+    println("Saved: descriptive_plot_peak_selection.png")
+end
+
+"""
+Visualizes the adaptive peak binning process, showing how peaks from different
+spectra are grouped into a common bin based on a PPM tolerance.
+"""
+function plot_peak_binning_details()
+    fig = Figure(size=(1200, 700), fontsize=14)
+    ax = Axis(fig[1, 1], title="Detailed View: Adaptive Peak Binning", xlabel="m/z", yticklabelsvisible=false)
+
+    ref_mz = 500.0
+    tolerance_ppm = 50.0
+    tol_mz = ref_mz * tolerance_ppm / 1e6
+    
+    bin_start = ref_mz - tol_mz/2
+    bin_end = ref_mz + tol_mz/2
+    
+    peaks_mz = [ref_mz - 0.01, ref_mz + 0.005, ref_mz + 0.02, ref_mz - 0.015]
+    peak_intensities = [0.8, 1.0, 0.9, 0.7]
+    peak_colors = [:blue, :green, :purple, :orange]
+    
+    vspan!(ax, bin_start, bin_end, color=(:gray, 0.2), label="Bin (tolerance: 50 ppm)")
+    stem!(ax, peaks_mz, peak_intensities, color=peak_colors, markersize=15)
+    
+    # Show bin center
+    bin_center = mean(peaks_mz)
+    vlines!(ax, [bin_center], color=:red, linestyle=:dash, label="Calculated Bin Center")
+
+    text!(ax, "Parameter: `tolerance`\nPeaks from different spectra within the tolerance window are grouped into a single feature.",
+          position=Point2f(0.05, 0.95), space=:relative, align=(:left, :top), fontsize=12)
+    axislegend(ax, position=:rt)
+    xlims!(ax, ref_mz - tol_mz*2, ref_mz + tol_mz*2)
+    save("descriptive_plot_peak_binning.png", fig)
+    println("Saved: descriptive_plot_peak_binning.png")
+end
+
+
+"""
+Main function to generate and save all descriptive plots.
+"""
+function create_and_save_all_plots()
+    println("Generating detailed descriptive plots for preprocessing steps...")
+    
+    plot_baseline_correction_details()
+    plot_smoothing_details()
+    plot_peak_picking_profile_details()
+    plot_peak_picking_centroid_details()
+    plot_normalization_details()
+    plot_calibration_details()
+    plot_alignment_details()
+    plot_peak_selection_details()
+    plot_peak_binning_details()
+
+    println("\nAll descriptive plots have been saved in the current directory.")
+end
+
+# Execute the plot generation
+if abspath(PROGRAM_FILE) == @__FILE__
+    create_and_save_all_plots()
+end
 
-# Display the plot (if in an interactive environment)
-fig

test/run_precalculation_example.jl

@@ -16,8 +16,9 @@ using MSI_src
 const TEST_MZML_FILE = ""
 # const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.imzML"
 #const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML"
-const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
+# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
 #const MASK_ROUTE = "/home/pixel/Documents/Cinvestav_2025/JuliaMSI/public/css/masks/Stomach_DHB_uncompressed.png"
+const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML"
 const MASK_ROUTE = ""
 
 #=

test/run_preprocessing.jl

@@ -5,6 +5,8 @@ import Pkg
 using CairoMakie
 using DataFrames    # For creating dataframes
 using CSV
+using Statistics
+using Interpolations
 
 # --- Load the MSI_src Module ---
 Pkg.activate(joinpath(@__DIR__, ".."))
@@ -15,9 +17,10 @@ using MSI_src
 # CONFIG: PLEASE FILL IN YOUR FILE PATHS HERE
 # ===================================================================
 
-const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
+# const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Thricoderma_etc/Imaging_interaccion_trichoderma_vs_streptomyces.imzML"
 # const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/set de datos MS/Atropina_tuneo_fraq_20ev.mzML"
 # const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML"
+const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML"
 
 # const MASK_ROUTE = "/home/pixel/Documents/Cinvestav_2025/JuliaMSI/public/css/masks/Stomach_DHB_uncompressed.png"
 const MASK_ROUTE = "" 
@@ -41,6 +44,55 @@ const reference_peaks = Dict(
     124.0393 => "Tropine [M+H]+",
 )
 
+const PIPELINE_STP = [
+    "stabilization",
+    #"baseline_correction",
+    "smoothing", 
+    "peak_picking",
+    "peak_selection",
+    #"calibration",
+    "peak_alignment",
+    "normalization",
+    "peak_binning"
+]
+
+# ===================================================================
+# USER OVERRIDES: Manually specify parameters here
+# ===================================================================
+# This dictionary allows you to override any auto-detected parameters.
+# The structure should match the output of `main_precalculation`.
+# Example: Force a less aggressive peak prominence threshold.
+
+const USER_OVERRIDES = Dict(
+    :Stabilization => Dict(
+        :method => :sqrt # Default stabilization method
+    ),
+    :PeakPicking => Dict(
+        #:snr_threshold => 1.5,
+        #:half_window => 5,
+        :snr_threshold => 8.0,
+        #:merge_peaks_tolerance => 2.5,
+        #:half_window => 2
+        #:half_window => 3
+    ),
+    # :Smoothing => Dict(
+    #     :window => 11
+    # )
+    #:PeakAlignment => Dict(
+    #    :tolerance => 0.01
+    #)
+    #:PeakSelection => Dict(
+        #:frequency_threshold => 0,
+        #:min_shape_r2 => 0.5,
+        #:max_fwhm_ppm => 30.0,
+        #:min_fwhm_ppm => 2.0
+    #),
+    #:PeakBinning => Dict(
+        #:tolerance => 30.0,
+        #:frequency_threshold => 0
+    #)
+)
+
 # ===================================================================
 # HELPER FUNCTIONS
 # ===================================================================
@@ -79,7 +131,7 @@ end
 # PREPROCESSING PIPELINE FUNCTIONS (IN-PLACE)
 # ===================================================================
 
-function apply_baseline_correction(spectra::Vector{MutableSpectrum}, params::Dict, msi_data::MSIData)
+function apply_baseline_correction_core(spectra::Vector{MutableSpectrum}, params::Dict, msi_data::MSIData)
     print_step_header("Baseline Correction")
     
     method = get(params, :method, :snip)
@@ -91,7 +143,7 @@ function apply_baseline_correction(spectra::Vector{MutableSpectrum}, params::Dic
     if !isempty(spectra)
         s = spectra[1]
         if validate_spectrum(s.mz, s.intensity)
-            baseline = MSI_src.apply_baseline_correction(s.intensity; method=method, iterations=iterations, window=window)
+            baseline = MSI_src.apply_baseline_correction_core(s.intensity; method=method, iterations=iterations, window=window)
             corrected_intensity = max.(0.0, s.intensity .- baseline)
             plot_spectrum_step(s.mz, corrected_intensity, "baseline_correction")
         end
@@ -99,7 +151,7 @@ function apply_baseline_correction(spectra::Vector{MutableSpectrum}, params::Dic
 
     Threads.@threads for s in spectra
         if validate_spectrum(s.mz, s.intensity)
-            baseline = MSI_src.apply_baseline_correction(s.intensity; method=method, iterations=iterations, window=window)
+            baseline = MSI_src.apply_baseline_correction_core(s.intensity; method=method, iterations=iterations, window=window)
             s.intensity = max.(0.0, s.intensity .- baseline)
         end
     end
@@ -117,14 +169,14 @@ function apply_smoothing(spectra::Vector{MutableSpectrum}, params::Dict)
     if !isempty(spectra)
         s = spectra[1]
         if validate_spectrum(s.mz, s.intensity)
-            smoothed_intensity = max.(0.0, smooth_spectrum(s.intensity; method=method, window=window, order=order))
+            smoothed_intensity = max.(0.0, smooth_spectrum_core(s.intensity; method=method, window=window, order=order))
             plot_spectrum_step(s.mz, smoothed_intensity, "smoothing", spectrum_index=s.id)
         end
     end
 
     Threads.@threads for s in spectra
         if validate_spectrum(s.mz, s.intensity)
-            smoothed_intensity = max.(0.0, smooth_spectrum(s.intensity; method=method, window=window, order=order))
+            smoothed_intensity = max.(0.0, smooth_spectrum_core(s.intensity; method=method, window=window, order=order))
             s.intensity = smoothed_intensity
         end
     end
@@ -153,13 +205,13 @@ function apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
 
         if is_valid
             if method == :profile
-                s.peaks = detect_peaks_profile(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window, min_peak_prominence=min_peak_prominence, merge_peaks_tolerance=merge_peaks_tolerance)
+                s.peaks = detect_peaks_profile_core(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window, min_peak_prominence=min_peak_prominence, merge_peaks_tolerance=merge_peaks_tolerance)
             elseif method == :wavelet
                 s.peaks = detect_peaks_wavelet(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
             elseif method == :centroid
-                s.peaks = detect_peaks_centroid(s.mz, s.intensity; snr_threshold=snr_threshold)
+                s.peaks = detect_peaks_centroid_core(s.mz, s.intensity; snr_threshold=snr_threshold)
             else
-                s.peaks = detect_peaks_profile(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
+                s.peaks = detect_peaks_profile_core(s.mz, s.intensity; snr_threshold=snr_threshold, half_window=half_window)
             end
         else
             s.peaks = []
@@ -176,6 +228,51 @@ function apply_peak_picking(spectra::Vector{MutableSpectrum}, params::Dict)
     end
 end
 
+function apply_peak_selection(spectra::Vector{MutableSpectrum}, params::Dict)
+    print_step_header("Peak Selection")
+    
+    min_snr = get(params, :min_snr, 0.0)
+    min_fwhm = get(params, :min_fwhm_ppm, 0.0)
+    max_fwhm = get(params, :max_fwhm_ppm, Inf)
+    min_r2 = get(params, :min_shape_r2, 0.0)
+    
+    # Handle `nothing` values from params, default to non-filtering values
+    min_snr = isnothing(min_snr) ? 0.0 : min_snr
+    min_fwhm = isnothing(min_fwhm) ? 0.0 : min_fwhm
+    max_fwhm = isnothing(max_fwhm) ? Inf : max_fwhm
+    min_r2 = isnothing(min_r2) ? 0.0 : min_r2
+
+    println("  - Min SNR: $min_snr")
+    println("  - FWHM Range (ppm): [$min_fwhm, $max_fwhm]")
+    println("  - Min Shape R²: $min_r2")
+
+    total_peaks_before = sum(s -> length(s.peaks), spectra)
+
+    Threads.@threads for s in spectra
+        if !isempty(s.peaks)
+            s.peaks = filter(p -> 
+                p.snr >= min_snr &&
+                (min_fwhm <= p.fwhm <= max_fwhm) &&
+                p.shape_r2 >= min_r2,
+                s.peaks
+            )
+        end
+    end
+
+    total_peaks_after = sum(s -> length(s.peaks), spectra)
+    println("  - Peaks before: $total_peaks_before, Peaks after: $total_peaks_after")
+
+    # Safely plot the first spectrum to show effect of filtering
+    if !isempty(spectra)
+        s1_idx = findfirst(s -> s.id == 1, spectra)
+        if s1_idx !== nothing
+            s1 = spectra[s1_idx]
+            plot_spectrum_step(s1.mz, s1.intensity, "peak_selection", peaks=s1.peaks, spectrum_index=s1.id)
+            println("  - Spectrum 1 now has $(length(s1.peaks)) peaks after selection.")
+        end
+    end
+end
+
 function apply_calibration(spectra::Vector{MutableSpectrum}, params::Dict, reference_peaks::Dict)
     print_step_header("Calibration")
     
@@ -195,7 +292,7 @@ function apply_calibration(spectra::Vector{MutableSpectrum}, params::Dict, refer
         s = spectra[i]
         info_message = ""
         if validate_spectrum(s.mz, s.intensity)
-            matched_peaks = find_calibration_peaks(s.mz, s.intensity, reference_masses; ppm_tolerance=ppm_tolerance)
+            matched_peaks = find_calibration_peaks_core(s.mz, s.intensity, reference_masses; ppm_tolerance=ppm_tolerance)
             if length(matched_peaks) >= 2
                 measured = sort(collect(values(matched_peaks)))
                 theoretical = sort(collect(keys(matched_peaks)))
@@ -248,7 +345,7 @@ function apply_peak_alignment(spectra::Vector{MutableSpectrum}, params::Dict)
         end
         
         current_peaks_mz = [p.mz for p in s.peaks]
-        alignment_func = align_peaks_lowess(ref_peaks_mz, current_peaks_mz; method=method, tolerance=tolerance, tolerance_unit=tolerance_unit)
+        alignment_func = align_peaks_lowess_core(ref_peaks_mz, current_peaks_mz; method=method, tolerance=tolerance, tolerance_unit=tolerance_unit)
         
         s.mz = alignment_func.(s.mz) # Update m/z axis
         
@@ -261,7 +358,7 @@ function apply_peak_alignment(spectra::Vector{MutableSpectrum}, params::Dict)
     end
 end
 
-function apply_normalization(spectra::Vector{MutableSpectrum}, params::Dict)
+function apply_normalization_core(spectra::Vector{MutableSpectrum}, params::Dict)
     print_step_header("Normalization")
     
     method = get(params, :method, :tic)
@@ -273,7 +370,7 @@ function apply_normalization(spectra::Vector{MutableSpectrum}, params::Dict)
         if s1 !== nothing
             s = spectra[s1]
             if validate_spectrum(s.mz, s.intensity)
-                normalized_intensity = MSI_src.apply_normalization(s.intensity; method=method)
+                normalized_intensity = MSI_src.apply_normalization_core(s.intensity; method=method)
                 plot_spectrum_step(s.mz, normalized_intensity, "normalization")
             end
         end
@@ -281,7 +378,7 @@ function apply_normalization(spectra::Vector{MutableSpectrum}, params::Dict)
 
     Threads.@threads for s in spectra
         if validate_spectrum(s.mz, s.intensity)
-            s.intensity = MSI_src.apply_normalization(s.intensity; method=method)
+            s.intensity = MSI_src.apply_normalization_core(s.intensity; method=method)
         end
     end
 end
@@ -293,8 +390,6 @@ function apply_peak_binning(spectra::Vector{MutableSpectrum}, params::Dict)
     tolerance = get(params, :tolerance, 20.0)
     tolerance_unit = get(params, :tolerance_unit, :ppm)
     min_peak_per_bin = get(params, :min_peak_per_bin, 3)
-    max_bin_width_ppm = get(params, :max_bin_width_ppm, 150.0)
-    intensity_weighted_centers = get(params, :intensity_weighted_centers, true)
     println("  Method: $method, Tolerance: $tolerance $tolerance_unit")
 
     if isempty(spectra) || all(s -> isempty(s.peaks), spectra)
@@ -304,93 +399,130 @@ function apply_peak_binning(spectra::Vector{MutableSpectrum}, params::Dict)
     
     println("  - Binning peaks from $(length(spectra)) spectra")
     
-    binning_params = PeakBinningParams(
+    # Collect all peaks with their intensities
-        method=method,
+    all_peaks = Vector{Tuple{Float64, Float64}}()  # (mz, intensity)
-        tolerance=tolerance,
+    for s in spectra
-        tolerance_unit=tolerance_unit,
+        for p in s.peaks
-        min_peak_per_bin=min_peak_per_bin,
+            push!(all_peaks, (p.mz, p.intensity))
-        max_bin_width_ppm=max_bin_width_ppm,
+        end
-        intensity_weighted_centers=intensity_weighted_centers
-    )
-    
-    feature_matrix, bin_definitions = bin_peaks(spectra, binning_params)
-    
-    if feature_matrix !== nothing && bin_definitions !== nothing
-        println("  - Generated feature matrix: $(size(feature_matrix.matrix))")
-        println("  - Number of bins: $(length(bin_definitions))")
     end
     
-    return feature_matrix, bin_definitions
+    if isempty(all_peaks)
+        @warn "No peaks collected for binning."
+        return nothing, nothing
+    end
+    
+    sort!(all_peaks, by=x->x[1])
+    
+    # Create bins - just store mz_center and intensity
+    bin_centers = Float64[]
+    bin_intensities = Float64[]
+    
+    i = 1
+    while i <= length(all_peaks)
+        current_bin_start = i
+        current_peak = all_peaks[i]
+        
+        # Find all peaks in this bin
+        j = i + 1
+        while j <= length(all_peaks)
+            next_peak = all_peaks[j]
+            # Calculate tolerance
+            tol = (tolerance_unit == :ppm) ? (current_peak[1] * tolerance / 1e6) : tolerance
+            
+            if (next_peak[1] - current_peak[1]) <= tol
+                j += 1
+            else
+                break
+            end
+        end
+        
+        current_bin_end = j - 1
+        bin_size = current_bin_end - current_bin_start + 1
+        
+        # Check if we have enough peaks in this bin
+        if bin_size >= min_peak_per_bin
+            bin_peaks_core = all_peaks[current_bin_start:current_bin_end]
+            
+            # Calculate m/z center and average intensity
+            mz_sum = 0.0
+            intensity_sum = 0.0
+            for peak in bin_peaks_core
+                mz_sum += peak[1]
+                intensity_sum += peak[2]
+            end
+            
+            mz_center = mz_sum / bin_size
+            avg_intensity = intensity_sum / bin_size
+            
+            push!(bin_centers, mz_center)
+            push!(bin_intensities, avg_intensity)
+        end
+        
+        i = j  # Move to next potential bin
+    end
+    
+    # Create the 2-row matrix
+    if !isempty(bin_centers)
+        n_bins = length(bin_centers)
+        feature_matrix = Matrix{Float64}(undef, 2, n_bins)
+        
+        for i in 1:n_bins
+            feature_matrix[1, i] = bin_centers[i]
+            feature_matrix[2, i] = bin_intensities[i]
+        end
+        
+        println("  - Created feature matrix: 2 × $n_bins")
+        println("  - Number of bins created: $n_bins")
+        println("  - m/z range: $(round(bin_centers[1], digits=4)) - $(round(bin_centers[end], digits=4))")
+        
+        # Return bin info as a vector of tuples for compatibility
+        bin_info = [(bin_centers[i], bin_intensities[i]) for i in 1:n_bins]
+        return feature_matrix, bin_info
+    else
+        @warn "No bins created after filtering"
+        return nothing, nothing
+    end
 end
 
-function save_feature_matrix(feature_matrix, bin_definitions)
+function save_feature_matrix(feature_matrix::Matrix{Float64}, bin_info)
     print_step_header("Saving Results")
     
-    # Save feature matrix as CSV
+    # Save as simple CSV with m/z and intensity rows
-    csv_path = joinpath(OUTPUT_DIR, "feature_matrix.csv")
+    csv_path = joinpath(OUTPUT_DIR, "feature_matrix_simple.csv")
     
-    # Create column headers
-    bin_headers = ["bin_$(i)_$(round(def[1], digits=4))-$(round(def[2], digits=4))" 
-                         for (i, def) in enumerate(bin_definitions)]
-    
-    # Open file for writing
     open(csv_path, "w") do io
-        # Write header row
+        # Write header
-        write(io, "spectrum_index," * join(bin_headers, ",") * "\n")
+        write(io, "mz,intensity\n")
         
-        # Write data rows
+        # Write data: m/z values in first column, intensities in second
-        for r_idx in 1:size(feature_matrix.matrix, 1)
+        for i in 1:size(feature_matrix, 2)
-            write(io, "$(feature_matrix.sample_ids[r_idx]),")
+            mz = feature_matrix[1, i]
-            for c_idx in 1:size(feature_matrix.matrix, 2)
+            intensity = feature_matrix[2, i]
-                write(io, "$(feature_matrix.matrix[r_idx, c_idx])")
+            write(io, "$mz,$intensity\n")
-                if c_idx < size(feature_matrix.matrix, 2)
-                    write(io, ",")
         end
     end
-            write(io, "\n")
+    println("  - Saved simple feature matrix: $csv_path")
-        end
-    end
-    println("  - Saved feature matrix: $csv_path")
     
-    # Save bin definitions (still using DataFrame as it's small)
+    # Also save in a more standard format for MSI
-    bins_path = joinpath(OUTPUT_DIR, "bin_definitions.csv")
+    csv_path_standard = joinpath(OUTPUT_DIR, "feature_matrix_standard.csv")
-    bins_df = DataFrame(
-        bin_index = 1:length(bin_definitions),
-        mz_start = [def[1] for def in bin_definitions],
-        mz_end = [def[2] for def in bin_definitions],
-        mz_center = [(def[1] + def[2])/2 for def in bin_definitions]
-    )
-    CSV.write(bins_path, bins_df)
-    println("  - Saved bin definitions: $bins_path")
     
-    return csv_path, bins_path
+    open(csv_path_standard, "w") do io
+        # Write header with m/z values as column names
+        write(io, "sample_type,")
+        mz_headers = [@sprintf("mz_%.4f", feature_matrix[1, i]) for i in 1:size(feature_matrix, 2)]
+        write(io, join(mz_headers, ",") * "\n")
+        
+        # Write the aggregated intensity values
+        write(io, "aggregated_spectrum,")
+        intensity_values = [feature_matrix[2, i] for i in 1:size(feature_matrix, 2)]
+        write(io, join(string.(intensity_values), ",") * "\n")
+    end
+    println("  - Saved standard format matrix: $csv_path_standard")
+    
+    return csv_path, csv_path_standard
 end
 
-# ===================================================================
-# USER OVERRIDES: Manually specify parameters here
-# ===================================================================
-# This dictionary allows you to override any auto-detected parameters.
-# The structure should match the output of `main_precalculation`.
-# Example: Force a less aggressive peak prominence threshold.
-const USER_OVERRIDES = Dict(
-    :PeakPicking => Dict(
-        #:snr_threshold => 1.5,
-        :half_window => 5,
-        :snr_threshold => 15.0,
-        #:merge_peaks_tolerance => 2.5,
-        #:half_window => 2
-    ),
-    # :Smoothing => Dict(
-    #     :window => 11
-    # )
-    #:PeakAlignment => Dict(
-    #    :tolerance => 0.01
-    #)
-    :PeakBinningParams => Dict(
-        :tolerance => 30.0
-    )
-)
-
 # ===================================================================
 # MAIN PREPROCESSING PIPELINE
 # ===================================================================
@@ -446,28 +578,45 @@ function run_preprocessing_pipeline()
     end
     
     # Define pipeline steps
-    pipeline_steps = [
+    pipeline_steps = PIPELINE_STP
-        "baseline_correction",
-        "smoothing", 
-        "peak_picking",
-        "calibration",
-        "peak_alignment",
-        "normalization",
-        "peak_binning"
-    ]
     
     println("\nPipeline steps: $(join(pipeline_steps, " -> "))")
     
     # Initialize `current_spectra` as a Vector of mutable structs for in-place modification
     println("\nInitializing spectra data structure...")
-    num_spectra = length(msi_data.spectra_metadata)
+    local spectrum_indices_to_process::AbstractVector{Int}
-    current_spectra = Vector{MutableSpectrum}(undef, num_spectra)
+    if !isempty(MASK_ROUTE)
-    _iterate_spectra_fast(msi_data) do idx, mz, intensity
+        println("Applying mask from: $(MASK_ROUTE)")
-        current_spectra[idx] = MutableSpectrum(idx, mz, intensity, [])
+        try
+            mask_matrix = MSI_src.load_and_prepare_mask(MASK_ROUTE, msi_data.image_dims)
+            masked_indices_set = MSI_src.get_masked_spectrum_indices(msi_data, mask_matrix)
+            spectrum_indices_to_process = collect(masked_indices_set)
+            println("Mask applied. $(length(spectrum_indices_to_process)) spectra are within the masked region.")
+        catch e
+            @error "Failed to load or apply mask: $e. Proceeding without mask."
+            spectrum_indices_to_process = 1:length(msi_data.spectra_metadata)
+        end
+    else
+        spectrum_indices_to_process = 1:length(msi_data.spectra_metadata)
+    end
 
-        # Plot raw spectrum without masks
+    if isempty(spectrum_indices_to_process)
-        if idx == 1
+        @warn "No spectra available for processing after applying mask/filter. Exiting pipeline."
-            plot_spectrum_step(mz, intensity, "raw_unmasked_spectrum")
+        close(msi_data)
+        return
+    end
+
+    num_spectra_to_process = length(spectrum_indices_to_process)
+    current_spectra = Vector{MutableSpectrum}(undef, num_spectra_to_process)
+    
+    Threads.@threads for i in 1:num_spectra_to_process
+        original_idx = spectrum_indices_to_process[i]
+        mz, intensity = MSI_src.GetSpectrum(msi_data, original_idx)
+        current_spectra[i] = MSI_src.MutableSpectrum(original_idx, mz, intensity, [])
+        
+        # Plot raw spectrum without masks (only the first processed one)
+        if i == 1
+            plot_spectrum_step(mz, intensity, "raw_unmasked_spectrum", spectrum_index=original_idx)
         end
     end
     
@@ -481,8 +630,33 @@ function run_preprocessing_pipeline()
         println("PROCESSING STEP: $step")
         println("-"^60)
         
-        if step == "baseline_correction"
+        if step == "stabilization"
-            @time apply_baseline_correction(current_spectra, auto_params[:BaselineCorrection], msi_data)
+            print_step_header("Intensity Transformation (Stabilization)")
+            method = get(auto_params[:Stabilization], :method, :sqrt)
+            println("  Method: $method")
+            
+            # Safely plot the first spectrum before transformation
+            if !isempty(current_spectra)
+                s = current_spectra[1]
+                if validate_spectrum(s.mz, s.intensity)
+                    # Use a temporary spectrum to plot before and after
+                    initial_intensity = deepcopy(s.intensity)
+                    plot_spectrum_step(s.mz, initial_intensity, "stabilization_before", spectrum_index=s.id)
+                end
+            end
+
+            @time apply_intensity_transformation(current_spectra, auto_params[:Stabilization])
+            
+            # Safely plot the first spectrum after transformation
+            if !isempty(current_spectra)
+                s = current_spectra[1]
+                if validate_spectrum(s.mz, s.intensity)
+                    plot_spectrum_step(s.mz, s.intensity, "stabilization_after", spectrum_index=s.id)
+                end
+            end
+            
+        elseif step == "baseline_correction"
+            @time apply_baseline_correction_core(current_spectra, auto_params[:BaselineCorrection], msi_data)
             
         elseif step == "smoothing"
             apply_smoothing(current_spectra, auto_params[:Smoothing])
@@ -490,6 +664,9 @@ function run_preprocessing_pipeline()
         elseif step == "peak_picking"
             @time apply_peak_picking(current_spectra, auto_params[:PeakPicking])
 
+        elseif step == "peak_selection"
+            @time apply_peak_selection(current_spectra, auto_params[:PeakSelection])
+            
         elseif step == "calibration"
             @time apply_calibration(current_spectra, auto_params[:Calibration], reference_peaks)
             
@@ -497,14 +674,14 @@ function run_preprocessing_pipeline()
             @time apply_peak_alignment(current_spectra, auto_params[:PeakAlignment])
             
         elseif step == "normalization"
-            @time apply_normalization(current_spectra, auto_params[:Normalization])
+            @time apply_normalization_core(current_spectra, auto_params[:Normalization])
             
         elseif step == "peak_binning"
             # This step is different as it generates the final matrix, not modifying spectra in-place
-            feature_matrix, bin_definitions = @time apply_peak_binning(current_spectra, auto_params[:PeakBinningParams])
+            feature_matrix, bin_info = @time apply_peak_binning(current_spectra, auto_params[:PeakBinning])
             
             if feature_matrix !== nothing
-                @time save_feature_matrix(feature_matrix, bin_definitions)
+                @time save_feature_matrix(feature_matrix, bin_info)
             end
             
         else

test/run_tests.jl

@@ -31,9 +31,10 @@ using MSI_src
 # --- Test Case 1: Standard .mzML file ---
 # A regular, non-imaging mzML file.
 # const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/mzML/T9_A1.mzML"
-const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML"
+# const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/CE4_BF_R1/CE4_BF_R1.mzML"
 # const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging_paper_spray/Imaging_paper_spray.mzML"
 # const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging prueba Roya 1/Roya.mzML"
+const TEST_MZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/mzML"
 const SPECTRUM_TO_PLOT = 1 # Which spectrum to plot from the file
 
 # --- Test Case 2: .mzML + Sync File for Conversion ---
@@ -58,16 +59,17 @@ const CONVERSION_TARGET_IMZML = "test/results/converted_mzml.imzML"
 # const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Imaging prueba Roya 1/royaimg.imzML"
 # const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/ltpmsi-chilli.imzML" # centroid aparently?
 # const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_compressed.imzML" # centroid compressed
-const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML" # centroid
+# const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/salida/Stomach_DHB_uncompressed.imzML" # centroid
+const TEST_IMZML_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML"
 # The m/z value to use for creating an image slice.
 # const MZ_VALUE_FOR_SLICE = 309.06 # BF
-# const MZ_VALUE_FOR_SLICE = 896.0 # HR2MSI
+const MZ_VALUE_FOR_SLICE = 896.0 # HR2MSI
 # const MZ_VALUE_FOR_SLICE = 76.03 # I PS
 # const MZ_VALUE_FOR_SLICE = 313 # ROYA
-const MZ_VALUE_FOR_SLICE = 100 # advanced processing
+# const MZ_VALUE_FOR_SLICE = 100 # advanced processing
 # const MZ_TOLERANCE = 0.1
 # const MZ_TOLERANCE = 1
-const MZ_TOLERANCE = 0.1
+const MZ_TOLERANCE = 0.2
 
 # Coordinates to plot a specific spectrum from imzML
 const COORDS_TO_PLOT = (50, 50) # Example coordinates (X, Y)

test/runtests.jl

@@ -0,0 +1,58 @@
+using Test
+
+# We need to simulate loading the MSI_src module directly
+include("../src/MSI_src.jl")
+using .MSI_src
+
+@testset "JuliaMSI Core Systems" begin
+    @testset "Basic Instantiation" begin
+        # 1. Test basic metadata structure initialization
+        meta = SpectrumMetadata(
+            Int32(1), Int32(2), 
+            "test_id", 
+            :sample, 
+            CENTROID, 
+            SpectrumAsset(Float64, false, Int64(0), 100, :mz, 0.0, 0.0), 
+            SpectrumAsset(Float32, true, Int64(100), 50, :intensity, 0.0, 0.0)
+        )
+        @test meta.x == 1
+        @test meta.y == 2
+        @test meta.mode == CENTROID
+        @test meta.mz_asset.format == Float64
+        @test meta.int_asset.format == Float32
+        @test meta.int_asset.is_compressed == true
+
+        # 2. Test cache pool and MSIData structural stability
+        source = MzMLSource([], Float64, Float32, nothing) # Empty source for structural testing
+        
+        # Test constructor doesn't throw
+        # Constructor signature: (source, metadata, instrument_meta, dims, coordinate_map, cache_size)
+        msi_data = MSIData(
+            source, 
+            [meta], 
+            nothing, # instrument_meta
+            (100, 100), # dims
+            nothing, # coord map
+            10 # cache size
+        )
+        
+        @test msi_data.image_dims == (100, 100)
+        @test length(msi_data.spectra_metadata) == 1
+        @test msi_data.cache_size == 10
+    end
+
+    @testset "Buffer & Cache Subsystems" begin
+        pool = SimpleBufferPool()
+        # Test basic retrieval
+        buf = get_buffer!(pool, 1024)
+        @test length(buf) == 1024
+        
+        # Test release
+        release_buffer!(pool, buf)
+        @test length(pool.buffers[1024]) == 1
+        
+        # Test reuse
+        buf2 = get_buffer!(pool, 1024)
+        @test buf === buf2 # Should return the EXACT same buffer object
+    end
+end

test/test_buffer_pool_concurrency.jl

@@ -0,0 +1,32 @@
+using MSI_src
+using Test
+using Base.Threads
+
+@testset "SimpleBufferPool Concurrency Stress Test" begin
+    pool = MSI_src.SimpleBufferPool()
+    n_threads = Threads.nthreads()
+    n_iterations = 1000
+    buffer_size = 1024
+
+    println("Running buffer pool stress test with $n_threads threads...")
+
+    # Parallel stress test
+    Threads.@threads for i in 1:(n_threads * n_iterations)
+        # get_buffer! and release_buffer! are now thread-safe
+        buf = MSI_src.get_buffer!(pool, buffer_size)
+        
+        # Simulate some work
+        fill!(buf, UInt8(i % 256))
+        
+        MSI_src.release_buffer!(pool, buf)
+    end
+
+    # After stress test, the dictionary should be coherent
+    sizes = collect(keys(pool.buffers))
+    if !isempty(sizes)
+        @test buffer_size ∈ sizes
+        @test length(pool.buffers[buffer_size]) <= pool.max_pool_size
+    end
+    
+    println("Buffer pool stress test PASSED.")
+end

test/test_pipeline_concurrency.jl

@@ -0,0 +1,146 @@
+
+using Pkg
+Pkg.activate(joinpath(@__DIR__, ".."))
+using MSI_src
+using Test
+using Base.Threads
+
+# --- Test Configuration ---
+const TEST_FILE = "/home/pixel/Documents/Cinvestav_2025/Analisis/Stomach/Stomach_DHB_uncompressed.imzML"
+
+println("Starting concurrency test with file: $TEST_FILE")
+
+if !isfile(TEST_FILE)
+    error("Test file not found: $TEST_FILE")
+end
+
+# --- Simulation of the Concurrency Issue ---
+
+# Global "UI State" variable simulates the app's msi_data
+global_msi_data = nothing
+const GLOBAL_LOCK = ReentrantLock()
+
+function ui_load_file(path)
+    global global_msi_data
+    lock(GLOBAL_LOCK) do
+        if global_msi_data !== nothing
+            close(global_msi_data)
+        end
+        println("UI: Loading file...")
+        global_msi_data = OpenMSIData(path)
+        println("UI: File loaded.")
+    end
+end
+
+function ui_close_file()
+    global global_msi_data
+    lock(GLOBAL_LOCK) do
+        if global_msi_data !== nothing
+            println("UI: Closing file...")
+            close(global_msi_data)
+            global_msi_data = nothing
+            println("UI: File closed.")
+        end
+    end
+end
+
+# Mock pipeline function that mimics `run_full_pipeline` in app.jl
+# Crucially, it uses its own local `pipeline_msi_data` as per the fix
+function run_mock_pipeline_isolated(path)
+    println("Pipeline: Starting isolated pipeline...")
+    
+    # 1. Open ISOLATED instance
+    pipeline_msi_data = OpenMSIData(path)
+    println("Pipeline: Opened isolated MSIData instance.")
+    
+    try
+        # 2. Simulate reading spectra
+        indices = 1:min(100, length(pipeline_msi_data.spectra_metadata)) # Read first 100 spectra
+        
+        # Artificial delay to allow "user" interaction
+        sleep(0.5) 
+        
+        println("Pipeline: Reading spectra...")
+        Threads.@threads for i in indices
+            # Use local instance
+            mz, int = GetSpectrum(pipeline_msi_data, i)
+            # Simulate processing work
+            sum(int) 
+        end
+        println("Pipeline: Finished reading spectra successfully.")
+        return true
+    catch e
+        println("Pipeline: CRASHED with error: $e")
+        return false
+    finally
+        close(pipeline_msi_data)
+        println("Pipeline: Closed isolated MSIData instance.")
+    end
+end
+
+# Mock pipeline that uses GLOBAL instance (The BUGGY version)
+function run_mock_pipeline_buggy()
+    println("Buggy Pipeline: Starting...")
+    # Uses global_msi_data directly
+    
+    try
+        global global_msi_data
+        if global_msi_data === nothing
+            println("Buggy Pipeline: No data loaded!")
+            return false
+        end
+        
+        local_ref = global_msi_data # Still points to same object
+        
+        indices = 1:min(100, length(local_ref.spectra_metadata))
+        
+        sleep(0.5)
+        
+        println("Buggy Pipeline: Reading spectra from shared object...")
+        Threads.@threads for i in indices
+            # This will fail if ui_close_file() happens concurrently
+            mz, int = GetSpectrum(local_ref, i)
+            sum(int)
+        end
+        println("Buggy Pipeline: Success (Unexpected if concurrency worked)")
+        return true
+    catch e
+        println("Buggy Pipeline: CRASHED as expected: $e")
+        return false
+    end
+end
+
+
+# --- execute checks ---
+
+@testset "Concurrency Crash Fix Verification" begin
+    
+    # 1. Setup: Load file initially
+    ui_load_file(TEST_FILE)
+    
+    # 2. Test the FIX: Isolated Pipeline
+    println("\n--- Testing Fixed (Isolated) Pipeline ---")
+    
+    t_pipeline = Threads.@spawn run_mock_pipeline_isolated(TEST_FILE)
+    
+    # Simulate user closing/reloading file while pipeline runs
+    sleep(0.2) 
+    ui_close_file()
+    
+    # Wait for pipeline
+    success = fetch(t_pipeline)
+    @test success == true
+    println("Fixed pipeline result: ", success ? "PASSED" : "FAILED")
+
+
+    # 3. Test the BUG: Global Pipeline (Optional, to prove it crashes without fix)
+    # Uncomment to verify the bug exists if needed, but we assume it does based on user report.
+    # println("\n--- Testing Buggy (Shared) Pipeline ---")
+    # ui_load_file(TEST_FILE)
+    # t_buggy = Threads.@spawn run_mock_pipeline_buggy()
+    # sleep(0.2)
+    # ui_close_file()
+    # buggy_success = fetch(t_buggy)
+    # println("Buggy pipeline result: ", buggy_success ? "PASSED (No crash?)" : "FAILED (Crashed as expected)")
+    
+end

test/test_streaming_pipeline.jl

@@ -0,0 +1,124 @@
+#!/usr/bin/env julia
+# test/test_streaming_pipeline.jl
+# ============================================================================
+# Validation test for Sprint 2: The Streaming Pipeline
+#
+# This test exercises process_dataset! against the HR2MSI mouse bladder
+# dataset and verifies:
+#   1. Correct sparse matrix creation
+#   2. Non-zero peak population
+#   3. RAM savings vs dense equivalent
+#   4. Allocation count and throughput
+# ============================================================================
+
+using Pkg
+Pkg.activate(".")
+
+using MSI_src
+using SparseArrays
+
+# =============================================================================
+# Configuration
+# =============================================================================
+
+const IMZML_PATH = "/home/pixel/Documents/Cinvestav_2025/Analisis/imzML_AP_SMALDI/HR2MSImouseurinarybladderS096.imzML"
+
+function main()
+    println("=" ^ 60)
+    println("SPRINT 2: Streaming Pipeline Validation")
+    println("=" ^ 60)
+    
+    if !isfile(IMZML_PATH)
+        println("SKIPPED: Dataset not found at $IMZML_PATH")
+        return
+    end
+    
+    # --- 1. Load dataset ---
+    println("\n--- Step 1: Loading dataset ---")
+    data = OpenMSIData(IMZML_PATH)
+    println("Loaded: $(length(data.spectra_metadata)) spectra")
+    
+    # --- 2. Configure the streaming pipeline ---
+    println("\n--- Step 2: Configuring pipeline ---")
+    config = PipelineConfig(
+        steps = [
+            StreamingStep(:baseline_correction, Dict{Symbol,Any}(:method => :snip, :iterations => 50)),
+            StreamingStep(:normalization, Dict{Symbol,Any}(:method => :tic)),
+            StreamingStep(:peak_picking, Dict{Symbol,Any}(
+                :method => :profile,
+                :snr_threshold => 3.0,
+                :half_window => 10,
+                :min_peak_prominence => 0.1,
+                :merge_peaks_tolerance => 0.002
+            )),
+        ],
+        num_bins = 2000,
+        frequency_threshold = 0.01  # Bins must appear in at least 1% of spectra
+    )
+    println("Steps: $(join([s.name for s in config.steps], " → "))")
+    println("Bins: $(config.num_bins), Frequency threshold: $(config.frequency_threshold)")
+    
+    # --- 3. Run the streaming pipeline ---
+    println("\n--- Step 3: Running streaming pipeline ---")
+    stats = @timed begin
+        feature_matrix, bin_centers = process_dataset!(data, config)
+    end
+    
+    feature_matrix = stats.value[1]
+    bin_centers = stats.value[2]
+    
+    println("\n--- Results ---")
+    println("  Feature matrix size: $(size(feature_matrix))")
+    println("  Non-zeros: $(nnz(feature_matrix))")
+    println("  Bin centers: $(length(bin_centers))")
+    println("  Time: $(round(stats.time, digits=2))s")
+    println("  Allocations: $(stats.bytes ÷ 1_000_000) MB")
+    println("  GC time: $(round(stats.gctime, digits=2))s")
+    
+    # --- 4. Validate ---
+    println("\n--- Step 4: Validation ---")
+    
+    passed = true
+    
+    # Check matrix dimensions
+    if size(feature_matrix, 1) > 0 && size(feature_matrix, 2) > 0
+        println("  ✓ Matrix has valid dimensions")
+    else
+        println("  ✗ Matrix has invalid dimensions: $(size(feature_matrix))")
+        passed = false
+    end
+    
+    # Check non-zeros
+    if nnz(feature_matrix) > 0
+        println("  ✓ Matrix has $(nnz(feature_matrix)) non-zero entries")
+    else
+        println("  ✗ Matrix is completely empty")
+        passed = false
+    end
+    
+    # Check sparsity savings
+    dense_mb = size(feature_matrix, 1) * size(feature_matrix, 2) * 8 / 1e6
+    sparse_mb = nnz(feature_matrix) * 16 / 1e6  # index + value per entry
+    if dense_mb > 0
+        savings = (1.0 - sparse_mb / dense_mb) * 100
+        println("  ✓ RAM savings: $(round(savings, digits=1))% ($(round(sparse_mb, digits=1)) MB vs $(round(dense_mb, digits=1)) MB dense)")
+    end
+    
+    # Check bin centers alignment
+    if length(bin_centers) == size(feature_matrix, 1)
+        println("  ✓ Bin centers match matrix rows")
+    else
+        println("  ✗ Bin center count ($(length(bin_centers))) != matrix rows ($(size(feature_matrix, 1)))")
+        passed = false
+    end
+    
+    println("\n" * "=" ^ 60)
+    if passed
+        println("ALL VALIDATIONS PASSED ✓")
+    else
+        println("SOME VALIDATIONS FAILED ✗")
+    end
+    println("=" ^ 60)
+end
+
+@time main()