65
65
},
66
66
"cursor_incremental_sync" : {
67
67
"type" : "DatetimeBasedCursor" ,
68
- "cursor_datetime_formats" : ["%Y-%m-%dT%H:%M:%SZ" , "%Y-%m-%dT%H:%M:%S%z" ],
68
+ "cursor_datetime_formats" : ["%Y-%m-%dT%H:%M:%SZ" , "%Y-%m-%dT%H:%M:%S%z" , "%ms" ],
69
69
"datetime_format" : "%Y-%m-%dT%H:%M:%SZ" ,
70
70
"cursor_field" : "{{ parameters.get('cursor_field', 'updated_at') }}" ,
71
71
"start_datetime" : {"datetime" : "{{ config.get('start_date')}}" },
@@ -399,13 +399,16 @@ def _run_read(
399
399
VOTE_200_CREATED_AT = "2024-01-12T00:00:00Z" # Latest vote in partition 20
400
400
VOTE_210_CREATED_AT = "2024-01-12T00:00:15Z" # Latest vote in partition 21
401
401
VOTE_300_CREATED_AT = "2024-01-10T00:00:00Z" # Latest vote in partition 30
402
+ VOTE_300_CREATED_AT_TIMESTAMP = 1704844800000 # Latest vote in partition 30
402
403
403
404
# Initial State Constants
404
405
PARENT_COMMENT_CURSOR_PARTITION_1 = "2023-01-04T00:00:00Z" # Parent comment cursor (partition)
405
406
PARENT_POSTS_CURSOR = "2024-01-05T00:00:00Z" # Parent posts cursor (expected in state)
406
407
407
408
INITIAL_STATE_PARTITION_10_CURSOR = "2024-01-02T00:00:01Z"
409
+ INITIAL_STATE_PARTITION_10_CURSOR_TIMESTAMP = 1704153601000
408
410
INITIAL_STATE_PARTITION_11_CURSOR = "2024-01-03T00:00:02Z"
411
+ INITIAL_STATE_PARTITION_11_CURSOR_TIMESTAMP = 1704240002000
409
412
INITIAL_GLOBAL_CURSOR = INITIAL_STATE_PARTITION_11_CURSOR
410
413
INITIAL_GLOBAL_CURSOR_DATE = datetime .fromisoformat (
411
414
INITIAL_STATE_PARTITION_11_CURSOR .replace ("Z" , "" )
@@ -596,7 +599,7 @@ def _run_read(
596
599
{
597
600
"id" : 300 ,
598
601
"comment_id" : 30 ,
599
- "created_at" : VOTE_300_CREATED_AT ,
602
+ "created_at" : VOTE_300_CREATED_AT_TIMESTAMP ,
600
603
}
601
604
]
602
605
},
@@ -637,7 +640,7 @@ def _run_read(
637
640
{
638
641
"comment_id" : 30 ,
639
642
"comment_updated_at" : COMMENT_30_UPDATED_AT ,
640
- "created_at" : VOTE_300_CREATED_AT ,
643
+ "created_at" : str ( VOTE_300_CREATED_AT_TIMESTAMP ) ,
641
644
"id" : 300 ,
642
645
},
643
646
],
@@ -662,7 +665,7 @@ def _run_read(
662
665
"id" : 10 ,
663
666
"parent_slice" : {"id" : 1 , "parent_slice" : {}},
664
667
},
665
- "cursor" : {"created_at" : INITIAL_STATE_PARTITION_10_CURSOR },
668
+ "cursor" : {"created_at" : INITIAL_STATE_PARTITION_10_CURSOR_TIMESTAMP },
666
669
},
667
670
{
668
671
"partition" : {
@@ -672,7 +675,7 @@ def _run_read(
672
675
"cursor" : {"created_at" : INITIAL_STATE_PARTITION_11_CURSOR },
673
676
},
674
677
],
675
- "state" : {"created_at" : INITIAL_STATE_PARTITION_11_CURSOR },
678
+ "state" : {"created_at" : INITIAL_STATE_PARTITION_11_CURSOR_TIMESTAMP },
676
679
"lookback_window" : 86400 ,
677
680
},
678
681
# Expected state
@@ -981,7 +984,15 @@ def run_incremental_parent_state_test(
981
984
# Fetch the first page of votes for comment 30 of post 3
982
985
(
983
986
f"https://api.example.com/community/posts/3/comments/30/votes?per_page=100&start_time={ LOOKBACK_DATE } " ,
984
- {"votes" : [{"id" : 300 , "comment_id" : 30 , "created_at" : VOTE_300_CREATED_AT }]},
987
+ {
988
+ "votes" : [
989
+ {
990
+ "id" : 300 ,
991
+ "comment_id" : 30 ,
992
+ "created_at" : VOTE_300_CREATED_AT_TIMESTAMP ,
993
+ }
994
+ ]
995
+ },
985
996
),
986
997
# Requests with intermediate states
987
998
# Fetch votes for comment 10 of post 1
@@ -1018,7 +1029,15 @@ def run_incremental_parent_state_test(
1018
1029
# Fetch votes for comment 30 of post 3
1019
1030
(
1020
1031
f"https://api.example.com/community/posts/3/comments/30/votes?per_page=100&start_time={ VOTE_300_CREATED_AT } " ,
1021
- {"votes" : [{"id" : 300 , "comment_id" : 30 , "created_at" : VOTE_300_CREATED_AT }]},
1032
+ {
1033
+ "votes" : [
1034
+ {
1035
+ "id" : 300 ,
1036
+ "comment_id" : 30 ,
1037
+ "created_at" : VOTE_300_CREATED_AT_TIMESTAMP ,
1038
+ }
1039
+ ]
1040
+ },
1022
1041
),
1023
1042
],
1024
1043
# Expected records
@@ -1056,7 +1075,7 @@ def run_incremental_parent_state_test(
1056
1075
{
1057
1076
"comment_id" : 30 ,
1058
1077
"comment_updated_at" : COMMENT_30_UPDATED_AT ,
1059
- "created_at" : VOTE_300_CREATED_AT ,
1078
+ "created_at" : str ( VOTE_300_CREATED_AT_TIMESTAMP ) ,
1060
1079
"id" : 300 ,
1061
1080
},
1062
1081
],
@@ -1344,7 +1363,15 @@ def test_incremental_parent_state(
1344
1363
(
1345
1364
f"https://api.example.com/community/posts/3/comments/30/votes"
1346
1365
f"?per_page=100&start_time={ PARTITION_SYNC_START_TIME } " ,
1347
- {"votes" : [{"id" : 300 , "comment_id" : 30 , "created_at" : VOTE_300_CREATED_AT }]},
1366
+ {
1367
+ "votes" : [
1368
+ {
1369
+ "id" : 300 ,
1370
+ "comment_id" : 30 ,
1371
+ "created_at" : VOTE_300_CREATED_AT_TIMESTAMP ,
1372
+ }
1373
+ ]
1374
+ },
1348
1375
),
1349
1376
],
1350
1377
# Expected records
@@ -1382,7 +1409,7 @@ def test_incremental_parent_state(
1382
1409
{
1383
1410
"comment_id" : 30 ,
1384
1411
"comment_updated_at" : COMMENT_30_UPDATED_AT ,
1385
- "created_at" : VOTE_300_CREATED_AT ,
1412
+ "created_at" : str ( VOTE_300_CREATED_AT_TIMESTAMP ) ,
1386
1413
"id" : 300 ,
1387
1414
},
1388
1415
],
@@ -1896,7 +1923,15 @@ def test_incremental_parent_state_no_records(
1896
1923
(
1897
1924
f"https://api.example.com/community/posts/3/comments/30/votes"
1898
1925
f"?per_page=100&start_time={ LOOKBACK_DATE } " ,
1899
- {"votes" : [{"id" : 300 , "comment_id" : 30 , "created_at" : VOTE_300_CREATED_AT }]},
1926
+ {
1927
+ "votes" : [
1928
+ {
1929
+ "id" : 300 ,
1930
+ "comment_id" : 30 ,
1931
+ "created_at" : VOTE_300_CREATED_AT_TIMESTAMP ,
1932
+ }
1933
+ ]
1934
+ },
1900
1935
),
1901
1936
],
1902
1937
# Expected records
@@ -1928,7 +1963,7 @@ def test_incremental_parent_state_no_records(
1928
1963
{
1929
1964
"comment_id" : 30 ,
1930
1965
"comment_updated_at" : COMMENT_30_UPDATED_AT ,
1931
- "created_at" : VOTE_300_CREATED_AT ,
1966
+ "created_at" : str ( VOTE_300_CREATED_AT_TIMESTAMP ) ,
1932
1967
"id" : 300 ,
1933
1968
},
1934
1969
],
0 commit comments