@@ -281,9 +281,21 @@ def to_configured_catalog(
281
281
},
282
282
"record_selector" : {
283
283
"type" : "RecordSelector" ,
284
- "extractor" : {"type" : "DpathExtractor" , "field_path" : []},
284
+ "extractor" : {"type" : "DpathExtractor" , "field_path" : ["data" ]},
285
+ },
286
+ "paginator" : {
287
+ "type" : "DefaultPaginator" ,
288
+ "page_token_option" : {
289
+ "type" : "RequestOption" ,
290
+ "inject_into" : "request_parameter" ,
291
+ "field_name" : "page_cursor" ,
292
+ },
293
+ "pagination_strategy" : {
294
+ "type" : "CursorPagination" ,
295
+ "cursor_value" : "{{ response.get('next_cursor') }}" ,
296
+ "stop_condition" : "{{ not response.get('has_more', False) }}" ,
297
+ },
285
298
},
286
- "paginator" : {"type" : "NoPagination" },
287
299
"partition_router" : {
288
300
"type" : "SubstreamPartitionRouter" ,
289
301
"parent_stream_configs" : [
@@ -543,10 +555,26 @@ def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_str
543
555
HttpRequest (url = f"https://api.test.com/parent/{ parent_id } /items" ),
544
556
HttpResponse (
545
557
body = json .dumps (
546
- [
547
- {"id" : 1 , "name" : "item_1" },
548
- {"id" : 2 , "name" : "item_2" },
549
- ]
558
+ {
559
+ "data" : [
560
+ {"id" : 1 , "name" : "item_1" },
561
+ ],
562
+ "has_more" : True ,
563
+ "next_cursor" : 1 ,
564
+ }
565
+ )
566
+ ),
567
+ )
568
+ http_mocker .get (
569
+ HttpRequest (url = f"https://api.test.com/parent/{ parent_id } /items?page_cursor=1" ),
570
+ HttpResponse (
571
+ body = json .dumps (
572
+ {
573
+ "data" : [
574
+ {"id" : 2 , "name" : "item_2" },
575
+ ],
576
+ "has_more" : False ,
577
+ }
550
578
)
551
579
),
552
580
)
0 commit comments