Rework api importer tests

.github/workflows/build.yaml

@@ -1,5 +1,8 @@
 name: Build
 on:
+  push:
+    branches: [main]
+
   pull_request:
     branches: [main]
 jobs:

.github/workflows/test.yaml

@@ -1,28 +1,27 @@
 name: Test
 on:
-  pull_request:
+  push:
     branches: [main]
+
+  pull_request:
+    branches:
+      - main
+
 jobs:
   test:
     runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    env:
-      POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/postgres
     steps:
-      - uses: actions/checkout@v4
-      - uses: oven-sh/setup-bun@v2
-      - run: bun install --frozen-lockfile
-      - run: bun run db:up
-      - run: bun run db:seed
-      - run: bun test
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Bun
+        uses: oven-sh/setup-bun@v2
+
+      - name: Set up Docker Compose
+        uses: docker/setup-buildx-action@v3
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Run tests
+        run: bun run test:ci

README.md

@@ -0,0 +1,69 @@
+# mohd-api
+
+Bun monorepo: GraphQL API + data importer for MOHD.
+
+## Quick Start
+
+```bash
+bun install
+bun run db:start
+bun run db:reset
+bun run dev
+```
+
+## Commands
+
+| Command | What |
+|---------|------|
+| `dev` | Start API locally |
+| `db:start` | Start Postgres |
+| `db:reset` | Import test data |
+| `db:stop` | Stop Postgres |
+| `test:service` | Run local tests |
+| `test:ci` | Run full test suite (CI) |
+| `import` | Run data importer |
+| `deploy:api` | Deploy API to Cloud Run |
+| `deploy:importer` | Deploy importer job |
+
+## Structure
+
+- `service/` - GraphQL API
+- `importer/` - Data import pipeline
+- `migrations/` - Docker setup + test data
+
+## Env
+
+Copy `.env.example` to `.env` in each package.
+
+Cloud Run needs:
+- `POSTGRES_URL` - DB connection string
+- `POSTGRES_PATH` - Cloud SQL socket (optional)
+
+## Cloud SQL Proxy
+
+Local access to Cloud SQL:
+
+```bash
+cloud-sql-proxy PROJECT:REGION:INSTANCE --port=5432
+```
+
+Set env:
+
+```
+POSTGRES_URL="postgresql://USER:PASSWORD@localhost:PORT/DATABASE"
+```
+
+## Importer
+
+```bash
+bun run import  # import all test data
+```
+
+Or manually:
+
+```bash
+cd importer
+bun run src/index.ts --datatype meta --schema public
+bun run src/index.ts --datatype rna --schema public
+bun run src/index.ts --datatype atac --schema public
+```

importer/src/atac.ts

@@ -2,7 +2,7 @@ import { streamImport } from "./utils";
 import { sql } from "./db";
 
 const filePath =
-  "https://users.wenglab.org/niship/Phase_0_ATAC_zscores.tsv";
+  process.env.ATAC_ZSCORE_FILE || "https://users.wenglab.org/niship/Phase_0_ATAC_zscores.tsv";
 
 export async function createATACTables() {
    await sql`

importer/src/meta.ts

@@ -1,11 +1,11 @@
 import { streamImport } from "./utils";
 import { sql } from "./db";
 
-const rnaFilePath = "https://users.wenglab.org/niship/Phase-0-Metadata.txt";
-const atacFilePath =
-  "https://users.wenglab.org/niship/Phase_0_ATAC_Metadata_with_entity.tsv";
+const rnaFilePath =  process.env.RNA_META_FILE  || "https://users.wenglab.org/niship/Phase-0-Metadata.txt";
+const atacFilePath =  process.env.ATAC_META_FILE  || "https://users.wenglab.org/niship/Phase_0_ATAC_Metadata_with_entity.tsv";
 
-const sexMap: Record<string, string> = { "1": "male", "2": "female" };
+
+const sexMap: Record<string, string> = { "1": "female", "2": "male" };
 
 export async function createMetaTables() {
 

importer/src/rna.ts

@@ -1,6 +1,6 @@
 import { streamImport } from "./utils";
 import { sql } from "./db";
-const filePath = "https://users.wenglab.org/niship/Phase-0_RNA-TPM.tsv";
+const filePath =  process.env.RNA_TPM_FILE || "https://users.wenglab.org/niship/Phase-0_RNA-TPM.tsv";
 
 export async function createRNATables() {
    await sql`

importer/src/utils.ts

@@ -6,8 +6,11 @@ export async function streamImport(
   parseLine: (line: string) => Record<string, string>,
 ) {
   console.log(`streaming ${table} from ${url}...`);
-  const response = await fetch(url);
-  const reader = response.body!.getReader();
+  //const response = await fetch(url);
+  //const reader = response.body!.getReader();
+
+  const reader = await getReader(url);
+
   const decoder = new TextDecoder();
 
   let buffer = "";
@@ -50,6 +53,34 @@ export async function streamImport(
   console.log(`inserted ${total} ${table} rows`);
 }
 
+async function getReader(source: string) {
+
+  // HTTP or HTTPS
+  if (source.startsWith("http://") || source.startsWith("https://")) {
+    const response = await fetch(source);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch ${source}: ${response.statusText}`);
+    }
+    return response.body!.getReader();
+  }
+
+  // file:// path
+  if (source.startsWith("file://")) {
+    const path = source.replace("file://", "");
+    const file = Bun.file(path);
+    return file.stream().getReader();
+  }
+
+  // assume local file path
+  const file = Bun.file(source);
+  if (!(await file.exists())) {
+    throw new Error(`Local file not found: ${source}`);
+  }
+
+  return file.stream().getReader();
+}
+
+
 async function insertRows(table: string, rows: Record<string, string>[]) {
   await sql`INSERT INTO ${sql(table)} ${sql(rows)}`;
 }

importer/tests/atac.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, test, beforeAll, afterAll } from "bun:test";
+import { SQL } from "bun";
+
+const schema = "test_schema_v1";
+
+const sql = new SQL({
+  url: "postgresql://postgres:postgres@postgres:5432/testdb",
+  max: 1,
+  connectionTimeout: 3,
+});
+
+beforeAll(async () => {
+  await sql`SET search_path TO ${sql(schema)}`;
+});
+afterAll(async () => {
+  await sql.end();
+});
+
+describe("ATAC ZScore Tables Integration Tests", () => {
+  test("Schema exists", async () => {
+    const result = await sql`
+      SELECT schema_name 
+      FROM information_schema.schemata 
+      WHERE schema_name = ${schema}
+    `;
+    expect(result.length).toBe(1);
+  });
+
+  test("atac_zscore table exists", async () => {
+    const result = await sql`
+      SELECT table_name
+      FROM information_schema.tables
+      WHERE table_schema = ${schema}
+        AND table_name = 'atac_zscore'
+    `;
+    expect(result.length).toBe(1);
+  });
+
+
+  test("ATAC ZScore was inserted", async () => {
+    const count = await sql`
+      SELECT COUNT(*)::int AS total
+      FROM ${sql(schema)}.atac_zscore
+    `;
+    expect(count[0].total).toBeGreaterThan(0);
+  });
+
+  test("First ATAC zscore accession is correct when ordered", async () => {
+    const result = await sql`
+      SELECT *
+      FROM ${sql(schema)}.atac_zscore ORDER BY accession LIMIT 1
+    `;        
+    expect(result[0].accession).toBe("EH38E0064571");   
+    expect(result[0].zscore_values.length).toBe(9);
+
+  });
+
+
+});