Performance optimization

.changeset/hip-years-cut.md

@@ -0,0 +1,10 @@
+---
+"@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": minor
+"@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv": minor
+"@platforma-open/milaboratories.immune-assay-data.workflow": minor
+"@platforma-open/milaboratories.immune-assay-data.model": minor
+"@platforma-open/milaboratories.immune-assay-data.ui": minor
+---
+
+- Introduce fast mode for sequence match
+- Support XLSX file as assay data input

model/src/index.ts

@@ -39,10 +39,14 @@ export type BlockArgs = {
   targetRef?: SUniversalPColumnId;
   fileHandle?: ImportFileHandle;
   fileExtension?: string;
+  detectedXsvType?: 'csv' | 'tsv';
   importColumns?: ImportColumnInfo[];
   sequenceColumnHeader?: string;
   selectedColumns: string[];
   settings: Settings;
+  lessSensitive: boolean;
+  mem?: number;
+  cpu?: number;
 };
 
 export type UiState = {
@@ -96,6 +100,7 @@ export const model = BlockModel.create()
       similarityType: 'alignment-score',
     },
     selectedColumns: [],
+    lessSensitive: false,
   })
 
   .withUiState<UiState>({

pnpm-workspace.yaml

@@ -3,6 +3,7 @@ packages:
   - software/add-header
   - software/coverage-mode-calc
   - software/fasta-to-tsv
+  - software/xlsx-to-csv
   - workflow
   - model
   - ui
@@ -37,6 +38,6 @@ catalog:
   # Block-specific dependencies
   '@milaboratories/software-pframes-conv': ^2.2.9
   "@platforma-open/milaboratories.runenv-python-3": ^1.7.5
-  "@platforma-open/soedinglab.software-mmseqs2": ^1.17.2
+  "@platforma-open/soedinglab.software-mmseqs2": 1.18.2
   '@biowasm/aioli': ~3.2.1
   "sass-embedded": ^1.77.8

software/coverage-mode-calc/src/main.py

@@ -42,7 +42,7 @@ def main():
                         help="Path for the output file (e.g., coverage_mode.txt).")
     args = parser.parse_args()
 
-    coverage_mode = "2"
+    coverage_mode = "1"
 
     clones_count, clones_total_length = get_fasta_stats(args.clones_fasta)
     assay_count, assay_total_length = get_fasta_stats(args.assay_fasta)
@@ -51,7 +51,7 @@ def main():
         clones_avg = clones_total_length / clones_count
         assay_avg = assay_total_length / assay_count
         if assay_avg < clones_avg:
-            coverage_mode = "1"
+            coverage_mode = "2"
 
     with open(args.output, 'w') as f:
         f.write(coverage_mode)

software/xlsx-to-csv/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv",
+  "version": "1.0.0",
+  "scripts": {
+    "build": "pl-pkg build",
+    "prepublishOnly": "pl-pkg prepublish",
+    "do-pack": "rm -f *.tgz && pl-pkg build && pnpm pack && mv platforma-open*.tgz package.tgz",
+    "changeset": "changeset",
+    "version-packages": "changeset version"
+  },
+  "files": [
+    "./dist/**/*"
+  ],
+  "dependencies": {},
+  "devDependencies": {
+    "@platforma-sdk/package-builder": "catalog:",
+    "@platforma-open/milaboratories.runenv-python-3": "catalog:"
+  },
+  "block-software": {
+    "entrypoints": {
+      "main": {
+        "binary": {
+          "artifact": {
+            "type": "python",
+            "registry": "platforma-open",
+            "environment": "@platforma-open/milaboratories.runenv-python-3:3.12.10",
+            "dependencies": {
+              "toolset": "pip",
+              "requirements": "requirements.txt"
+            },
+            "root": "./src"
+          },
+          "cmd": [
+            "python",
+            "{pkg}/main.py"
+          ]
+        }
+      }
+    }
+  }
+}

software/xlsx-to-csv/src/main.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Convert XLSX files to CSV format."""
+
+import argparse
+import csv
+import sys
+
+from openpyxl import load_workbook
+
+
+def find_header_row(rows: list) -> int:
+    """Find the header row index by looking for the first row where most cells are non-empty."""
+    for i, row in enumerate(rows):
+        non_empty = sum(1 for cell in row if cell is not None and str(cell).strip())
+        if non_empty > 1:
+            return i
+    return 0
+
+
+def xlsx_to_csv(input_file: str, output_file: str) -> None:
+    """Read the first worksheet of an XLSX file and write it as CSV."""
+    wb = load_workbook(input_file, read_only=True, data_only=True)
+    ws = wb[wb.sheetnames[0]]
+
+    rows = list(ws.iter_rows(values_only=True))
+    header_idx = find_header_row(rows)
+
+    with open(output_file, 'w', newline='') as f:
+        writer = csv.writer(f)
+        for row in rows[header_idx:]:
+            writer.writerow(
+                ['' if cell is None else cell for cell in row]
+            )
+
+    wb.close()
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Convert XLSX to CSV.")
+    parser.add_argument("-i", "--input", required=True, help="Input XLSX file path.")
+    parser.add_argument("-o", "--output", required=True, help="Output CSV file path.")
+    args = parser.parse_args()
+
+    try:
+        xlsx_to_csv(args.input, args.output)
+        print(f"Successfully converted '{args.input}' to '{args.output}'")
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

software/xlsx-to-csv/src/requirements.txt

@@ -0,0 +1 @@
+openpyxl

ui/src/importFile.ts

@@ -147,6 +147,7 @@ export async function importFile(file: LocalImportFileHandle) {
   const fileName = getFileNameFromHandle(file);
   const extension = fileName.split('.').pop()?.toLowerCase();
   app.model.args.fileExtension = extension;
+  app.model.args.detectedXsvType = undefined;
 
   let rawData: TableData;
 
@@ -165,6 +166,7 @@ export async function importFile(file: LocalImportFileHandle) {
   } else {
     // Handle Excel/CSV files as before
     const data = await getRawPlatformaInstance().lsDriver.getLocalFileContent(file);
+
     const wb = XLSX.read(data);
 
     // @TODO: allow user to select worksheet
@@ -175,6 +177,14 @@ export async function importFile(file: LocalImportFileHandle) {
       raw: true,
       blankrows: false,
     }) as TableData;
+
+    // Detect actual delimiter (extension may not match content).
+    // XLSX auto-detects internally via guess_sep but doesn't expose the result,
+    // so we check the first line of the already-in-memory data buffer.
+    if (extension === 'csv' || extension === 'tsv') {
+      const firstLine = new TextDecoder().decode(new Uint8Array(data).slice(0, 4096)).split('\n')[0] ?? '';
+      app.model.args.detectedXsvType = firstLine.includes('\t') ? 'tsv' : 'csv';
+    }
   }
 
   const header = rawData[0];

ui/src/pages/MainPage.vue

@@ -15,6 +15,8 @@ import {
   PlAgDataTableV2,
   PlBlockPage,
   PlBtnGhost,
+  PlAccordionSection,
+  PlCheckbox,
   PlDropdown,
   PlDropdownMulti,
   PlDropdownRef,
@@ -23,6 +25,7 @@ import {
   PlNumberField,
   PlSectionSeparator,
   PlSlideModal,
+  PlTooltip,
   usePlDataTableSettingsV2,
 } from '@platforma-sdk/ui-vue';
 import strings from '@milaboratories/strings';
@@ -244,7 +247,7 @@ const similarityTypeOptions = [
       </PlDropdown>
       <PlFileInput
         v-model="app.model.args.fileHandle" label="Assay data to import" placeholder="Assay data table"
-        :extensions="['csv', 'tsv', 'fasta', 'fa']" :error="app.model.ui.fileImportError" required @update:model-value="setFile"
+        :extensions="['csv', 'tsv', 'fasta', 'fa', 'xlsx']" :error="app.model.ui.fileImportError" required @update:model-value="setFile"
       >
         <template #tooltip>
           Upload a comma-separated (.csv), tab-separated (.tsv), or FASTA (.fasta/.fa) file containing assay data. FASTA files will be converted to a table with Header and Sequence columns.
@@ -304,6 +307,40 @@ const similarityTypeOptions = [
           Select min fraction of aligned (covered) residues of clonotypes in the cluster.
         </template>
       </PlNumberField>
+
+      <PlAccordionSection :label="strings.titles.advancedSettings">
+        <PlCheckbox v-model="app.model.args.lessSensitive">
+          Fast mode
+          <PlTooltip class="info" position="top">
+            <template #tooltip>Prioritizes speed over sensitivity. Reduces prefiltering precision, which may miss some weaker matches but significantly speeds up alignment for large datasets.</template>
+          </PlTooltip>
+        </PlCheckbox>
+
+        <PlSectionSeparator>Resource allocation</PlSectionSeparator>
+        <PlNumberField
+          v-model="app.model.args.mem"
+          label="Memory (GiB)"
+          :min-value="1"
+          :step="1"
+          :max-value="1012"
+        >
+          <template #tooltip>
+            Sets the amount of memory to use for the alignment.
+          </template>
+        </PlNumberField>
+
+        <PlNumberField
+          v-model="app.model.args.cpu"
+          label="CPU (cores)"
+          :min-value="1"
+          :step="1"
+          :max-value="128"
+        >
+          <template #tooltip>
+            Sets the number of CPU cores to use for the alignment.
+          </template>
+        </PlNumberField>
+      </PlAccordionSection>
     </PlSlideModal>
     <PlSlideModal
       v-model="multipleSequenceAlignmentAssayOpen"

workflow/package.json

@@ -14,6 +14,7 @@
     "@platforma-open/milaboratories.immune-assay-data.add-header": "workspace:*",
     "@platforma-open/milaboratories.immune-assay-data.coverage-mode-calc": "workspace:*",
     "@platforma-open/milaboratories.immune-assay-data.fasta-to-tsv": "workspace:*",
+    "@platforma-open/milaboratories.immune-assay-data.xlsx-to-csv": "workspace:*",
     "@platforma-open/soedinglab.software-mmseqs2": "catalog:"
   },
   "devDependencies": {

workflow/src/build-outputs.tpl.tengo

@@ -191,7 +191,7 @@ self.body(func(inputs) {
 	cloneImportResults := xsv.importFile(
 		inputs.clonesDataTsv, "tsv", {
 			axes: [{
-				column: "query",
+				column: "target",
 				spec: inputs.datasetSpec.axesSpec[1]
 			}],
 			columns: cloneColumns,
@@ -206,11 +206,11 @@ self.body(func(inputs) {
 		inputs.bestAlignmentTsv, "tsv", {
 			axes: [
 				{
-					column: "query",
+					column: "target",
 					spec: inputs.datasetSpec.axesSpec[1]
 				},
 				{
-					column: "target",
+					column: "query",
 					spec: {
 						name: "pl7.app/vdj/assay/sequenceId",
 						type: "String",