Merge pull request #14 from Snowflake-Labs/antlr-parser

Added test module

README.md

@@ -4,25 +4,24 @@
 
 ---
 
-DLSync is a database change management that deploys database changes to our database. 
-Each object(view, table, udf ...) in our database will 
-have a corresponding SQL script file where every change to this object is tracked in this file only. DLSync keeps track of what changes have been deployed to database 
+DLSync is a database change management tool designed to streamline the development and deployment of snowflake changes. 
+By associating each database object(view, table, udf ...) with a corresponding SQL script file, DLSync tracks every modification, ensuring efficient and accurate updates.
+Each script can also have a corresponding test script that can be used to write unit tests for the database object. 
+. DLSync keeps track of what changes have been deployed to database 
 by using hash. Hence DLSync is capable of identifying what scripts have changed in the current deployment.
 Using this DLSync only deploys changed script to database objects.
 DLSync also understands interdependency between different scripts, thus applies these changes
 according their dependency.
-Based on how we define the changes to database objects, DLSync divides database object scripts to 2 types, State and migration scripts.
 ## Key Features 
-- It combines state based and migration based change management to manage database changes
-- Each object will have it's corresponding unique Script file where we can define the change to the object
-- It can detect change between previous deployment and current script state.
-- It can reorder scripts based on their dependency before deploying to database.
-- It supports parametrization of scripts where we can define variables that change between different database instances.
-- It supports parameter config file where each parameter config file corresponds to an instance 
-- It supports rollback to previous deployment state. 
-- Rollback is very simple and intuitive. Only one needs to rollback git repository of the script and triggering rollback module.
-- It supports verify module where each database object is checked with current script to check for deployment verification or tracking out of sync database changes.
-- It supports create script where we can create script file for each database objects.
+- Hybrid Change Management: It combines state based and migration based change management to manage database changes
+- Unique Script per object: Each object will have it's corresponding unique Script file where we can define the change to the object
+- Unit Testing: It supports unit testing where we can write test scripts for each database object.
+- Change detection: It can detect change between previous deployment and current script state.
+- Dependency resolution: It can reorder scripts based on their dependency before deploying to database.
+- Parametrization: It supports parametrization of scripts where we can define variables that change between different database instances. Each instance is associated with parameter config file, where each parameter config lists the variables and their value for that instance. 
+- Rollback: It supports rollback to previous deployment state. Rollback is very simple and intuitive. Only one needs to rollback git repository of the script and triggering rollback module.
+- Verification: It supports verify module where each database object is checked with current script to check for deployment verification or tracking out of sync database changes.
+- Script creation: It supports create script where we can create script file for each database objects.
 
 ## Project structure
 To use this tool first create your script root directory.
@@ -47,6 +46,15 @@ Inside this directory create a directory structure like:
 │   │   │   │   ├── object_name_7.sql               # The database object name(table name, view name, function name ...)
 │   │   │   │   ├── object_name_8.sql               # The database object name(table name, view name, function name ...)
 ├── /tests                                          # SQL unit test scripts
+│   ├── /database_name_1                            
+│   │   ├── /schema_name_1                          
+│   │   │   ├── /[object_type]_1                    
+│   │   │   │   ├── object_name_1_test.sql          # unit test file for object object_name_1_test
+│   │   │   │   ├── object_name_2_test.sql          # unit test file for object object_name_2_test
+│   │   ├── /schema_name_2                          
+│   │   │   ├── /[object_type]_1                    
+│   │   │   │   ├── object_name_5_test.sql          # unit test file for object object_name_5_test
+│   │   │   │   ├── object_name_6_test.sql          # unit test file for object object_name_6_test
 ├── config.yml                                      # configuration file
 ├── parameter-[profile-1].properties                # parameter property file  
 ├── parameter-[profile-2].properties                # parameter property file

backlog.md

@@ -1,8 +1,4 @@
 # DLSync Backlog
-- [x] Rollback for migration
-- [x] Verify module State Script
-- [x] create script to capture config tables
-- [x] Script hierarchy design
-- [x] Verify module for migration Script
-- [ ] Migration Script parsing using ATLR
-- [ ] Support for different DB
+- [ ] use config file for connection properties
+- [ ] create command line application
+- [ ] use antlr4 for verify module

build.gradle

@@ -21,6 +21,7 @@ dependencies {
     implementation 'org.slf4j:slf4j-api:2.0.4'
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2'
     implementation 'commons-cli:commons-cli:1.9.0'
+    implementation 'org.antlr:antlr4-runtime:4.13.2'
 
     compileOnly 'org.projectlombok:lombok:1.18.24'
     annotationProcessor 'org.projectlombok:lombok:1.18.24'
@@ -48,4 +49,4 @@ jar {
 
 test {
     useJUnitPlatform()
-}
+}

example_scripts/main/example_db/MAIN_SCHEMA/FUNCTIONS/GET_RETURNED_ORDERS.SQL

@@ -0,0 +1,8 @@
+CREATE OR REPLACE FUNCTION ${EXAMPLE_DB}.${MAIN_SCHEMA}.GET_RETURNED_ORDERS(USER VARCHAR)
+RETURNS NUMERIC(10, 2)
+LANGUAGE SQL
+AS
+$$
+    SELECT COUNT(*) FROM ${EXAMPLE_DB}.${MAIN_SCHEMA}.ORDER_RETURNS
+    WHERE USER_ID = USER
+$$

example_scripts/main/example_db/MAIN_SCHEMA/TABLES/ORDER_RETURNS.SQL

@@ -0,0 +1,10 @@
+---version: 0, author: DlSync
+CREATE TABLE ${EXAMPLE_DB}.${MAIN_SCHEMA}.ORDER_RETURNS (
+    ID INT AUTOINCREMENT PRIMARY KEY,
+    USER_ID INT REFERENCES ${EXAMPLE_DB}.${MAIN_SCHEMA}.USERS(ID),
+    ORDER_ID INT REFERENCES ${EXAMPLE_DB}.${MAIN_SCHEMA}.ORDERS(ID),
+    REASON VARCHAR,
+    RETURN_DATE TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+---rollback: DROP TABLE IF EXISTS ${EXAMPLE_DB}.${MAIN_SCHEMA}.ORDER_RETURNS;
+---verify: SELECT * FROM ${EXAMPLE_DB}.${MAIN_SCHEMA}.ORDER_RETURNS LIMIT 1;

example_scripts/test/example_db/MAIN_SCHEMA/FUNCTIONS/CALCULATE_DISCOUNT_TEST.SQL

@@ -0,0 +1,7 @@
+with MOCK_DATA AS (
+    SELECT 100 AS P_PRICE, 10 AS P_DISCOUNT_RATE
+),
+EXPECTED_DATA AS (
+    SELECT 90 AS RETURN_VALUE
+)
+SELECT CALCULATE_ORDER_TOTAL(P_PRICE, P_DISCOUNT_RATE) AS RETURN_VALUE from MOCK_DATA;

example_scripts/test/example_db/MAIN_SCHEMA/FUNCTIONS/GET_RETURNED_ORDERS_TEST.SQL

@@ -0,0 +1,10 @@
+WITH MOCK_DATA AS (
+    SELECT 1 AS USER
+),
+ORDER_RETURNS AS (
+    SELECT * FROM VALUES(1, 1, 1, 'broken'), (2, 1, 2, 'not arrived'), (3, 2, 3, 'quality') AS T(ID, USER_ID, ORDER_ID, RETURNED_QUANTITY)
+),
+EXPECTED_DATA AS (
+    SELECT 2 AS RETURN_VALUE
+)
+SELECT GET_RETURNED_ORDERS(USER) AS RETURN_VALUE from MOCK_DATA;

example_scripts/test/example_db/MAIN_SCHEMA/VIEWS/STOCK_SUMMARY_TEST.SQL

@@ -0,0 +1,7 @@
+WITH PRODUCTS AS (
+    SELECT * FROM VALUES(1, 'Product 1', 5, 100), (2, 'Product 2', 20, 50), (3, 'Product 3', 30, 5) AS T(ID, PRODUCT_NAME, STOCK, PRICE)
+),
+EXPECTED_DATA AS (
+    SELECT * FROM VALUES(1, 'Product 1', 5, 'LOW STOCK'), (2, 'Product 2', 20, 'SUFFICIENT STOCK'), (3, 'Product 3', 30, 'SUFFICIENT STOCK') AS T(ID, PRODUCT_NAME, STOCK, STOCK_STATUS)
+)
+SELECT * FROM STOCK_SUMMARY;

example_scripts/test/example_db/MAIN_SCHEMA/VIEWS/USER_ORDER_SUMMARY_TEST.SQL

@@ -0,0 +1,13 @@
+WITH USERS AS (
+    SELECT * FROM VALUES(1, 'Alice'), (2, 'Bob') AS T(ID, USER_NAME)
+),
+ORDERS AS (
+    SELECT * FROM VALUES(1, 1, 1, 10), (2, 1, 2, 20), (3, 2, 3, 30) AS T(ID, USER_ID, PRODUCT_ID, QUANTITY)
+),
+PRODUCTS AS (
+    SELECT * FROM VALUES(1, 'Product 1', 100), (2, 'Product 2', 50), (3, 'Product 3', 5) AS T(ID, PRODUCT_NAME, PRICE)
+),
+EXPECTED_DATA AS (
+    SELECT * FROM VALUES(1, 'Alice', 2, 2000), (2, 'Bob', 1, 150) AS T(USER_ID, USER_NAME, TOTAL_ORDERS, TOTAL_SPENT)
+)
+SELECT * FROM ${EXAMPLE_DB}.${MAIN_SCHEMA}.USER_ORDER_SUMMARY;

gradle.properties

@@ -1 +1 @@
-releaseVersion=1.5.0
+releaseVersion=2.0.0

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.snowflake</groupId>
     <artifactId>dlsync</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <version>2.0-SNAPSHOT</version>
 
     <properties>
         <maven.compiler.source>11</maven.compiler.source>
@@ -57,6 +57,11 @@
             <artifactId>commons-cli</artifactId>
             <version>1.9.0</version>
         </dependency>
+        <dependency>
+            <groupId>antlr4-runtime</groupId>
+            <artifactId>org.antlr</artifactId>
+            <version>4.13.2</version>
+        </dependency>
 
         <dependency>
             <groupId>org.junit.jupiter</groupId>