Lambda-Miner: Current tool status

.github/chemical_formula_library/Dockerfile

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: 2024 Helmholtz Centre for Environmental Research GmbH - UFZ
+#
+# SPDX-License-Identifier: LicenseRef-UFZ-GPL-3.0-or-later
+
+# Use an official PostgreSQL image as the base image
+FROM postgres:13
+
+# Set environment variables
+ENV POSTGRES_DB=cflib
+ENV POSTGRES_USER=cflib_adm
+ENV POSTGRES_PASSWORD=BJSeVYMK6QaDE3eVfJEB
+
+# Copy scripts and data
+COPY ddl /home/cflib/ddl
+COPY data /home/cflib/data
+#COPY journal /home/cflib/journal
+
+# Copy the build script into the initdb entrypoint for automatic execution
+COPY build.sql /docker-entrypoint-initdb.d/
+
+# Expose the default PostgreSQL port
+EXPOSE 5432

.github/chemical_formula_library/README.md

@@ -0,0 +1,60 @@
+<!--
+SPDX-FileCopyrightText: 2025 Helmholtz Centre for Environmental Research GmbH - UFZ
+
+SPDX-License-Identifier: CC-BY-SA-4.0
+-->
+
+# Chemical Formula Library
+
+## Execution
+
+```bash
+# Build the image from the Dockerfile in .
+sudo docker build -t cflib_image .
+```
+
+Run the container listening to a different port (e.g. 5431) if you want to run
+it in parallel with another database container (e.g. lmdb container). Remember
+to also change any port settings to the new port.
+
+```bash
+# Run a container from the image
+sudo docker run -p 5431:5432 --name cflib_container -d cflib_image
+```
+
+Please be aware, that importing all the data will take a little time depending
+on the capacities of the executing machine (few minutes). The database is not
+reachable via `psql` during import. The command will result in an error.
+
+```bash
+# Connect to the database via the container
+sudo docker exec -it cflib_container psql -U cflib_adm cflib
+```
+
+```bash
+# Connect to the database in the container directly
+psql -h localhost -p 5431 -U cflib_adm cflib
+```
+
+```bash
+# Stop the container
+sudo docker stop cflib_container
+```
+
+```bash
+# Start an already stopped container
+sudo docker start cflib_container
+```
+
+```bash
+# Remove the container
+sudo docker rm cflib_container
+```
+
+## Database User
+
+Admin User: `cflib_adm` with password `BJSeVYMK6QaDE3eVfJEB`
+
+Read-Write User: `cflib_rw` with password `SSeTjhTE42ba9kgjDrUY`
+
+Read-Only User: `cflib_ro` with password `z7zDjfZbTQq8QU6dTHGL`

.github/chemical_formula_library/build.sql

@@ -0,0 +1,22 @@
+-- SPDX-FileCopyrightText: 2024 Helmholtz Centre for Environmental Research GmbH - UFZ
+--
+-- SPDX-License-Identifier: LicenseRef-UFZ-GPL-3.0-or-later
+
+/*
+*   This script builds the Lambda-Miner Database
+*/
+
+\i /home/cflib/ddl/cflib_schema.sql
+\i /home/cflib/ddl/cflib_privileges.sql
+
+/*
+*   Data
+*/
+
+\i /home/cflib/data/chemical_formula.sql
+
+/*
+*   Journal (recent changes)
+*/
+
+--\i /home/lmdb/journal/

.github/chemical_formula_library/cflib_local_config.json

@@ -0,0 +1,11 @@
+{
+    "host":"localhost",
+    "port":5431,
+    "database":"cflib",
+    "schema":"cflib",
+    "user":"cflib_adm",
+    "password":"BJSeVYMK6QaDE3eVfJEB",
+    "parameters":{
+        "sslmode":"disable"
+    }
+}

.github/chemical_formula_library/cflib_local_config.json.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2025 Helmholtz Centre for Environmental Research GmbH - UFZ
+
+SPDX-License-Identifier: LicenseRef-UFZ-GPL-3.0-or-later

.github/chemical_formula_library/ddl/cflib_privileges.sql

@@ -0,0 +1,34 @@
+-- SPDX-FileCopyrightText: 2024 Helmholtz Centre for Environmental Research GmbH - UFZ
+--
+-- SPDX-License-Identifier: LicenseRef-UFZ-GPL-3.0-or-later
+
+--
+-- Roles
+--
+
+-- Read-Write role
+CREATE ROLE read_write;
+GRANT CONNECT ON DATABASE cflib TO read_write;
+GRANT USAGE ON SCHEMA public TO read_write;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO read_write;
+
+-- Read-only role
+CREATE ROLE read_only;
+GRANT CONNECT ON DATABASE cflib TO read_only;
+GRANT USAGE ON SCHEMA public TO read_only;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO read_only;
+
+--
+-- Users
+--
+
+-- Administrator
+GRANT ALL PRIVILEGES ON DATABASE cflib TO cflib_adm;
+
+-- Read-Write user
+CREATE USER cflib_rw WITH ENCRYPTED PASSWORD 'SSeTjhTE42ba9kgjDrUY';
+GRANT read_write TO cflib_rw;
+
+-- Read-Only user
+CREATE USER cflib_ro WITH ENCRYPTED PASSWORD 'z7zDjfZbTQq8QU6dTHGL';
+GRANT read_only TO cflib_ro;

.github/chemical_formula_library/ddl/cflib_schema.sql

@@ -0,0 +1,168 @@
+-- SPDX-FileCopyrightText: 2024 Helmholtz Centre for Environmental Research GmbH - UFZ
+--
+-- SPDX-License-Identifier: LicenseRef-UFZ-GPL-3.0-or-later
+
+-- Dumped from database version 13.10
+-- Dumped by pg_dump version 14.11 (Ubuntu 14.11-0ubuntu0.22.04.1)
+
+--
+-- Name: cflib; Type: SCHEMA; Schema: -; Owner: cflib_adm
+--
+
+CREATE SCHEMA cflib;
+
+ALTER SCHEMA cflib OWNER TO cflib_adm;
+
+--
+-- Name: chemical_formula; Type: TABLE; Schema: cflib; Owner: cflib_adm
+--
+
+CREATE TABLE cflib.chemical_formula (
+    chemical_formula_id bigint NOT NULL,
+    exact_mass numeric(13,8) NOT NULL,
+    hc_ratio numeric(6,3),
+    nc_ratio numeric(6,3),
+    oc_ratio numeric(6,3),
+    pc_ratio numeric(6,3),
+    sc_ratio numeric(6,3),
+    dbe numeric(6,3),
+    dbe_o numeric(6,3),
+    formula_json json NOT NULL,
+    monoisotopic_parent bigint NOT NULL,
+    CONSTRAINT chemical_formula_exact_mass_check CHECK ((exact_mass > (0)::numeric)),
+    CONSTRAINT chemical_formula_hc_ratio_check CHECK ((hc_ratio >= (0)::numeric)),
+    CONSTRAINT chemical_formula_nc_ratio_check CHECK ((nc_ratio >= (0)::numeric)),
+    CONSTRAINT chemical_formula_oc_ratio_check CHECK ((oc_ratio >= (0)::numeric)),
+    CONSTRAINT chemical_formula_pc_ratio_check CHECK ((pc_ratio >= (0)::numeric)),
+    CONSTRAINT chemical_formula_sc_ratio_check CHECK ((sc_ratio >= (0)::numeric))
+);
+
+ALTER TABLE cflib.chemical_formula OWNER TO cflib_adm;
+
+--
+-- Name: do_assignments_for_peak(numeric[], numeric[], numeric[], numeric[], numeric[], numeric[], numeric[], numeric[], json); Type: FUNCTION; Schema: cflib; Owner: cflib_adm
+--
+
+CREATE FUNCTION cflib.do_assignments_for_peak(mass_range numeric[], rrange_hc numeric[], rrange_nc numeric[], rrange_oc numeric[], rrange_pc numeric[], rrange_sc numeric[], rrange_dbe numeric[], rrange_dbe_o numeric[], element_ranges json) RETURNS SETOF cflib.chemical_formula
+    LANGUAGE plpgsql PARALLEL SAFE
+    AS $$
+
+DECLARE
+cf_row cflib.chemical_formula%ROWTYPE;
+range json;
+range_c integer[];
+range_13c integer[];
+range_h integer[];
+range_2h integer[];
+range_n integer[];
+range_15n integer[];
+range_o integer[];
+range_18o integer[];
+range_p integer[];
+range_s integer[];
+range_34s integer[];
+range_cl integer[];
+range_37cl integer[];
+range_na integer[];
+
+BEGIN
+
+/* read the element ranges */
+SELECT COALESCE(element_ranges->'C', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_c;
+SELECT COALESCE(element_ranges->'13C', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_13c;
+SELECT COALESCE(element_ranges->'H', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_h;
+SELECT COALESCE(element_ranges->'2H', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_2h;
+SELECT COALESCE(element_ranges->'N', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_n;
+SELECT COALESCE(element_ranges->'15N', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_15n;
+SELECT COALESCE(element_ranges->'O', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_o;
+SELECT COALESCE(element_ranges->'18O', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_18o;
+SELECT COALESCE(element_ranges->'P', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_p;
+SELECT COALESCE(element_ranges->'S', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_s;
+SELECT COALESCE(element_ranges->'34S', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_34s;
+SELECT COALESCE(element_ranges->'Cl', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_cl;
+SELECT COALESCE(element_ranges->'37Cl', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_37cl;
+SELECT COALESCE(element_ranges->'Na', '[0,0]') INTO range;
+SELECT ARRAY[((range->0)::text)::integer, ((range->1)::text)::integer] INTO range_na;
+
+/* do the assignment */
+FOR cf_row IN
+SELECT *
+FROM cflib.chemical_formula cf
+
+/*
+following statements are less time consuming than the blow
+approx. 15 % of the total execution time
+*/
+WHERE cf.exact_mass BETWEEN mass_range[1] AND mass_range[2]
+AND cf.hc_ratio BETWEEN rrange_hc[1] AND rrange_hc[2]
+AND cf.nc_ratio BETWEEN rrange_nc[1] AND rrange_nc[2]
+AND cf.oc_ratio BETWEEN rrange_oc[1] AND rrange_oc[2]
+AND cf.pc_ratio BETWEEN rrange_pc[1] AND rrange_pc[2]
+AND cf.sc_ratio BETWEEN rrange_sc[1] AND rrange_sc[2]
+AND cf.dbe BETWEEN rrange_dbe[1] AND rrange_dbe[2]
+AND cf.dbe_o BETWEEN rrange_dbe_o[1] AND rrange_dbe_o[2]
+
+/*
+following statements are more time consuming than the above
+approx. 9/10 of the total execution time
+filter from infrequent to frequent elements/isotopes to reduce execution times
+*/
+AND COALESCE(((cf.formula_json->0->'Na')::text)::integer, 0) BETWEEN range_na[1] AND range_na[2]
+AND COALESCE(((cf.formula_json->0->'2H')::text)::integer, 0) BETWEEN range_2h[1] AND range_2h[2]
+AND COALESCE(((cf.formula_json->0->'37Cl')::text)::integer, 0) BETWEEN range_37cl[1] AND range_37cl[2]
+AND COALESCE(((cf.formula_json->0->'34S')::text)::integer, 0) BETWEEN range_34s[1] AND range_34s[2]
+AND COALESCE(((cf.formula_json->0->'18O')::text)::integer, 0) BETWEEN range_18o[1] AND range_18o[2]
+AND COALESCE(((cf.formula_json->0->'15N')::text)::integer, 0) BETWEEN range_15n[1] AND range_15n[2]
+AND COALESCE(((cf.formula_json->0->'13C')::text)::integer, 0) BETWEEN range_13c[1] AND range_13c[2]
+AND COALESCE(((cf.formula_json->0->'Cl')::text)::integer, 0) BETWEEN range_cl[1] AND range_cl[2]
+AND COALESCE(((cf.formula_json->0->'P')::text)::integer, 0) BETWEEN range_p[1] AND range_p[2]
+AND COALESCE(((cf.formula_json->0->'S')::text)::integer, 0) BETWEEN range_s[1] AND range_s[2]
+AND COALESCE(((cf.formula_json->0->'N')::text)::integer, 0) BETWEEN range_n[1] AND range_n[2]
+AND COALESCE(((cf.formula_json->0->'O')::text)::integer, 0) BETWEEN range_o[1] AND range_o[2]
+AND COALESCE(((cf.formula_json->0->'H')::text)::integer, 0) BETWEEN range_h[1] AND range_h[2]
+AND COALESCE(((cf.formula_json->0->'C')::text)::integer, 0) BETWEEN range_c[1] AND range_c[2]
+
+LOOP
+RETURN NEXT cf_row;
+END LOOP;
+END;
+$$;
+
+ALTER FUNCTION cflib.do_assignments_for_peak(mass_range numeric[], rrange_hc numeric[], rrange_nc numeric[], rrange_oc numeric[], rrange_pc numeric[], rrange_sc numeric[], rrange_dbe numeric[], rrange_dbe_o numeric[], element_ranges json) OWNER TO cflib_adm;
+
+--
+-- Name: chemical_formula chemical_formula_pkey; Type: CONSTRAINT; Schema: cflib; Owner: lmdb_adm
+--
+
+ALTER TABLE cflib.chemical_formula ADD CONSTRAINT chemical_formula_pkey PRIMARY KEY (chemical_formula_id);
+
+--
+-- Name: chemical_formula_exact_mass_idx; Type: INDEX; Schema: cflib; Owner: lmdb_adm
+--
+
+CREATE INDEX chemical_formula_exact_mass_idx ON cflib.chemical_formula USING btree (exact_mass);
+
+--
+-- Name: chemical_formula_monoisotopic_parent_idx; Type: INDEX; Schema: cflib; Owner: lmdb_adm
+--
+
+CREATE INDEX chemical_formula_monoisotopic_parent_idx ON cflib.chemical_formula USING btree (monoisotopic_parent);
+
+--
+-- Name: chemical_formula chemical_formula_monoisotopic_parent_fkey; Type: FK CONSTRAINT; Schema: cflib; Owner: lmdb_adm
+--
+
+ALTER TABLE cflib.chemical_formula ADD CONSTRAINT chemical_formula_monoisotopic_parent_fkey FOREIGN KEY (monoisotopic_parent) REFERENCES cflib.chemical_formula(chemical_formula_id) ON DELETE CASCADE;

.github/chemical_formula_library/snippets/240117-0_cflib-big_fix-partition-range.sql

@@ -0,0 +1,23 @@
+-- SPDX-FileCopyrightText: 2024 Helmholtz Centre for Environmental Research GmbH - UFZ
+--
+-- SPDX-License-Identifier: LicenseRef-UFZ-GPL-3.0-or-later
+
+/*
+* This script fixes the partition range of the big cflib partition 314. Data could not be inserted
+* into this partition, because the wanted id was out of range - the lower bound was set incorrectly
+* (156905591000).
+*
+* Author: Johann Wurz
+* Vendor: Helmholtz Centre for Environmental Research GmbH - UFZ
+*/
+
+BEGIN;
+
+-- Detach the problematic partition
+ALTER TABLE cflib.chemical_formula DETACH PARTITION cflib.chemical_formula_part_314;
+
+-- Attach the partition with a new lower bound
+ALTER TABLE cflib.chemical_formula ATTACH PARTITION cflib.chemical_formula_part_314 FOR VALUES FROM (156905590000) TO (157399766000);
+
+ROLLBACK;
+--COMMIT;

.github/lambdaminer/lmdb_local_config.json.license

@@ -1,3 +1,3 @@
-SPDX-FileCopyrightText: 2024 Helmholtz Centre for Environmental Research GmbH - UFZ
+SPDX-FileCopyrightText: 2025 Helmholtz Centre for Environmental Research GmbH - UFZ
 
 SPDX-License-Identifier: LicenseRef-UFZ-GPL-3.0-or-later

.github/workflows/ci.yaml

@@ -159,13 +159,21 @@ jobs:
           singularity pull --dir /tmp docker://rocker/tidyverse
         fi
 
-    # Set up the lambda-miner database
-    - name: Create a database in the postgres for lambda-miner
+    # Set up the Lambda-Miner database
+    - name: Create LMDB
       run: |
-        PGPASSWORD=postgres psql -h localhost -U postgres -c "CREATE DATABASE lmdb;"
-        PGPASSWORD=postgres psql -h localhost -U postgres -d lmdb -q -f ./.github/lambdaminer/build.sql
+        PGPASSWORD=postgres psql -h localhost -U postgres -p 5432 -c "CREATE DATABASE lmdb;"
+        PGPASSWORD=postgres psql -h localhost -U postgres -p 5432 -d lmdb -q -f ./.github/lambdaminer/build.sql
         sleep 10
-        PGPASSWORD=postgres psql -h localhost -U postgres -d lmdb -c "SELECT * FROM public.instrument;"
+        PGPASSWORD=postgres psql -h localhost -U postgres -p 5432 -d lmdb -c "SELECT * FROM public.instrument;"
+
+    # Set up the Chemical Formula Library for the Lambda-Miner
+    - name: Create CFLib
+      run: |
+        PGPASSWORD=postgres psql -h localhost -U postgres -p 5431 -c "CREATE DATABASE cflib;"
+        PGPASSWORD=postgres psql -h localhost -U postgres -p 5431 -d cflib -q -f ./.github/chemical_formula_library/build.sql
+        sleep 10
+        PGPASSWORD=postgres psql -h localhost -U postgres -p 5431 -d cflib -c "SELECT * FROM cflib.chemical_formula LIMIT 10;"
 
       # Start OMERO
     - name: Start OMERO