start testing ibm db2 compatibility

Author: windiana42

.github/workflows/ci.yml

@@ -38,6 +38,18 @@ jobs:
       matrix:
         os: [Ubuntu]
         python-version: ['3.8', '3.9', '3.10']
+    services:
+      ibm_db2:
+        image: ibmcom/db2:11.5.5.1
+        env:
+          LICENSE: accept
+          DB2INSTANCE: db2inst1
+          DB2INST1_PASSWORD: password
+          DBNAME: testdb
+          UPDATEAVAIL: "NO"
+        options: --privileged
+        ports:
+          - 50000:50000
     steps:
       - name: Checkout branch
         uses: actions/[email protected]
@@ -83,7 +95,7 @@ jobs:
 
       - name: Install dependencies
         shell: bash
-        run: poetry install -vvv
+        run: poetry install -vvv -E mssql -E ibm_db2
 
       - name: Start Docker Compose
         uses: isbang/[email protected]
@@ -94,8 +106,12 @@ jobs:
         shell: bash
         run: sleep 2
 
+      - name: Wait for DB/2
+        timeout-minutes: 2
+        run: until docker logs "${{ job.services.ibm_db2.id }}" 2>&1 | grep -q "Setup has completed"; do sleep 10; echo Waiting; done
+
       - name: Test
         shell: bash
         run: |
-          # poetry run pytest -q tests
-          poetry run pytest -s tests  # interactive log output
+          poetry run pytest -q tests
+          # poetry run pytest -s tests  # interactive log output

README.md

@@ -65,6 +65,19 @@ You can inspect the contents of the PipeDAT Postgres database at `postgresql://p
 To reset the state of the docker containers you can run `docker compose down`.
 This might be necessary if the database cache gets corrupted.
 
+## Testing db2 functionality
+
+For running @pytest.mark.ibm_db2 tests, you need to spin up a docker container without `docker compose` since it needs
+the `--priviledged` option which `docker compose` does not offer.
+
+```bash
+docker run -h db2server --name db2server --restart=always --detach --privileged=true -p 50000:50000 --env-file docker_db2.env_list -v /Docker:/database ibmcom/db2
+```
+
+Then check `docker logs db2server | grep -i completed` until you see `(*) Setup has completed.`.
+
+Afterwards you can run `pytest --ibm_db2`.
+
 ## Troubleshooting
 
 ### Installing mssql odbc driver for linux

docker_db2.env_list

@@ -0,0 +1,15 @@
+LICENSE=accept
+DB2INSTANCE=db2inst1
+DB2INST1_PASSWORD=password
+DBNAME=testdb
+BLU=false
+ENABLE_ORACLE_COMPATIBILITY=false
+UPDATEAVAIL=NO
+TO_CREATE_SAMPLEDB=false
+REPODB=false
+IS_OSXFS=false
+PERSISTENT_HOME=true
+HADR_ENABLED=false
+ETCD_ENDPOINT=
+ETCD_USERNAME=
+ETCD_PASSWORD=

pyproject.toml

@@ -41,6 +41,9 @@ sphinx-rtd-theme = { version = "^1.0.0", optional = true }
 sphinxcontrib-apidoc = { version="^0.3.0", optional = true }
 python-box = "^6.1.0"
 
+pyodbc = { version = "^4.0.35", optional = true }
+ibm-db = { version = "^3.1.4", optional = true }
+ibm-db-sa = { version = "^0.3.8", optional = true }
 prefect = { version = ">=1.3,<2", optional = true }
 # for prefect 2 (radical change of paradigmns especially in task graph display => not ideal for rather linear pipeline)
 # prefect = { version = "prefect>=2.0,<3", optional = true }
@@ -50,6 +53,7 @@ filelock = ["filelock"]
 zookeeper = ["kazoo"]
 docs = ["Sphinx", "sphinx-rtd-theme", "sphinxcontrib-apidoc"]
 mssql = ["pyodbc"]
+ibm_db2 = ["ibm-db", "ibm-db-sa"]
 prefect = ["prefect", "docker-compose"]
 
 [tool.poetry.dev-dependencies]

tests/test_db2.py

@@ -0,0 +1,29 @@
+import ibm_db
+import pandas as pd
+import sqlalchemy as sa
+import structlog
+
+
+def test_db2():
+    logger = structlog.getLogger(module=__name__)
+    conn = ibm_db.connect(
+        "DATABASE=testdb;HOSTNAME=localhost;PORT=50000;PROTOCOL=TCPIP;UID=db2inst1;PWD=password;",
+        "",
+        "",
+    )
+    query = "SELECT 1 as a FROM SYSIBM.SYSDUMMY1"
+    stmt = ibm_db.exec_immediate(conn, query)
+    row = True
+    rows = []
+    while row is not False:
+        row = ibm_db.fetch_assoc(stmt)
+        print(row)
+        logger.info("DB2 test row", row=row)
+        rows.append(row)
+
+
+def test_db2_sqlalchemy():
+    logger = structlog.getLogger(module=__name__)
+    engine = sa.create_engine("db2+ibm_db://db2inst1:password@localhost:50000/testdb")
+    df = pd.read_sql("SELECT 1 as a FROM SYSIBM.SYSDUMMY1", con=engine)
+    logger.info("DB2 test df", df="\n" + str(df))