Skip to content

Commit 6406fa4

Browse files
dolfinusdolfinus
committed
[DOP-26806] Merge ancestors and parents into one relation
1 parent b706db9 commit 6406fa4

10 files changed

Lines changed: 23 additions & 111 deletions

File tree

‎data_rentgen/server/schemas/v1/lineage.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,6 @@ class IndirectLineageColumnRelationV1(BaseModel):
236236

237237
class LineageRelationsResponseV1(BaseModel):
238238
parents: list[LineageParentRelationV1] = Field(description="Parent relations", default_factory=list)
239-
ancestors: list[LineageParentRelationV1] = Field(description="Ancestors relations", default_factory=list)
240239
symlinks: list[LineageSymlinkRelationV1] = Field(description="Symlink relations", default_factory=list)
241240
inputs: list[LineageInputRelationV1] = Field(description="Input relations", default_factory=list)
242241
outputs: list[LineageOutputRelationV1] = Field(description="Input relations", default_factory=list)

‎data_rentgen/server/utils/lineage_response.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,10 @@ def build_lineage_response(lineage: LineageServiceResult) -> LineageResponseV1:
5858
operations=operations, # type: ignore[assignment, arg-type]
5959
),
6060
relations=LineageRelationsResponseV1(
61-
parents=_get_run_parent_relations(lineage.runs) + _get_operation_parent_relations(lineage.operations),
62-
ancestors=_get_jobs_hierarchy_chain(lineage.job_ancestor_relations)
63-
+ _get_runs_hierarchy_chain(lineage.run_ancestor_relations),
61+
parents=_get_jobs_ancestor_relations(lineage.job_ancestor_relations)
62+
+ _get_run_ancestor_relations(lineage.run_ancestor_relations)
63+
+ _get_run_parent_relations(lineage.runs)
64+
+ _get_operation_parent_relations(lineage.operations),
6465
symlinks=_get_symlink_relations(lineage.dataset_symlinks),
6566
inputs=_get_input_relations(lineage.inputs),
6667
outputs=_get_output_relations(lineage.outputs),
@@ -334,7 +335,7 @@ def _get_datasets_with_dataset_granularity(
334335
return datasets
335336

336337

337-
def _get_runs_hierarchy_chain(runs_relations: set[tuple[UUID, UUID]]) -> list[LineageParentRelationV1]:
338+
def _get_run_ancestor_relations(runs_relations: set[tuple[UUID, UUID]]) -> list[LineageParentRelationV1]:
338339
parents = []
339340
for parent_run_id, run_id in runs_relations:
340341
relation = LineageParentRelationV1(
@@ -345,7 +346,7 @@ def _get_runs_hierarchy_chain(runs_relations: set[tuple[UUID, UUID]]) -> list[Li
345346
return sorted(parents, key=lambda x: (x.from_.id, x.to.id))
346347

347348

348-
def _get_jobs_hierarchy_chain(jobs_relations: set[tuple[int, int]]) -> list[LineageParentRelationV1]:
349+
def _get_jobs_ancestor_relations(jobs_relations: set[tuple[int, int]]) -> list[LineageParentRelationV1]:
349350
parents = []
350351
for parent_job_id, job_id in jobs_relations:
351352
relation = LineageParentRelationV1(

‎docs/changelog/next_release/392.improvement.rst‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,6 @@ Added ``ancestors`` field to lineage ``relations`` response. It contains run→r
2828
"parents": [
2929
{"from": {"kind": "JOB", "id": "1"}, "to": {"kind": "RUN", "id": "run-uuid"}},
3030
{"from": {"kind": "RUN", "id": "run-uuid"}, "to": {"kind": "OPERATION", "id": "op-uuid"}}
31-
],
32-
"ancestors": [
3331
{"from": {"kind": "RUN", "id": "parent-run-uuid"}, "to": {"kind": "RUN", "id": "run-uuid"}}
3432
],
3533
"symlinks": [],

‎docs/entities/index.rst‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ Entities
3535
[Operation] --> [Run]: PARENT
3636
[Run] --> [User]: started by
3737
[Run] --> [Run]: PARENT
38+
[Job] --> [Job]: PARENT
3839
[Dataset1] ..> [Operation]: INPUT
3940
[Operation] ..> [Dataset1]: OUTPUT
4041

@@ -309,15 +310,14 @@ Parent Relation
309310

310311
Relation between child run/operation and its parent. For example:
311312

312-
- Spark applicationName is parent for all its runs (applicationId).
313-
- Spark applicationId is parent for all its Spark job or Spark execution.
313+
- Spark job (applicationName) is parent for all its runs (applicationId).
314314
- Airflow DAG is parent of Airflow task.
315-
- Airflow Task Instance triggered a Spark applicationId, dbt run, and so on.
315+
- Airflow Task Instance can start a Spark run (applicationId), dbt run, and so on.
316316

317317
It contains following fields:
318318

319319
- ``from: Job | Run`` - parent entity.
320-
- ``to: Run | Operation`` - child entity.
320+
- ``to: Job | Run | Operation`` - child entity.
321321

322322
.. image:: parent.png
323323

‎tests/test_server/test_lineage/test_column_lineage.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ async def test_get_dataset_lineage_with_empty_column_lineage(
7070
assert response.json() == {
7171
"relations": {
7272
"parents": run_parents_to_json(runs),
73-
"ancestors": [],
7473
"symlinks": [],
7574
"inputs": [
7675
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -129,7 +128,6 @@ async def test_get_operation_lineage_include_columns_with_combined_transformatio
129128
assert response.json() == {
130129
"relations": {
131130
"parents": run_parents_to_json([run]) + operation_parents_to_json([operation]),
132-
"ancestors": [],
133131
"symlinks": [],
134132
"inputs": [
135133
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -223,7 +221,6 @@ async def test_get_run_lineage_include_columns_with_combined_transformations(
223221
assert response.json() == {
224222
"relations": {
225223
"parents": run_parents_to_json([run]),
226-
"ancestors": [],
227224
"symlinks": [],
228225
"inputs": [
229226
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -316,7 +313,6 @@ async def test_get_job_lineage_include_columns_with_combined_transformations(
316313
assert response.json() == {
317314
"relations": {
318315
"parents": [],
319-
"ancestors": [],
320316
"symlinks": [],
321317
"inputs": inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
322318
"outputs": outputs_to_json(merge_io_by_jobs(outputs), granularity="JOB"),
@@ -441,7 +437,6 @@ async def test_get_dataset_lineage_include_columns_with_depth_and_granularity_ru
441437
assert response.json() == {
442438
"relations": {
443439
"parents": run_parents_to_json(runs),
444-
"ancestors": [],
445440
"symlinks": [],
446441
"inputs": [
447442
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -564,7 +559,6 @@ async def test_get_dataset_lineage_include_columns_with_depth_and_granularity_jo
564559
assert response.json() == {
565560
"relations": {
566561
"parents": [],
567-
"ancestors": [],
568562
"symlinks": [],
569563
"inputs": inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
570564
"outputs": outputs_to_json(merge_io_by_jobs(outputs), granularity="JOB"),
@@ -694,7 +688,6 @@ async def test_get_dataset_lineage_include_columns_with_depth_and_granularity_op
694688
assert response.json() == {
695689
"relations": {
696690
"parents": run_parents_to_json(runs) + operation_parents_to_json(operations),
697-
"ancestors": [],
698691
"symlinks": [],
699692
"inputs": [
700693
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -833,7 +826,6 @@ async def test_get_operation_lineage_include_columns_with_depth(
833826
assert response.json() == {
834827
"relations": {
835828
"parents": run_parents_to_json(runs) + operation_parents_to_json(operations),
836-
"ancestors": [],
837829
"symlinks": [],
838830
"inputs": [
839831
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -994,7 +986,6 @@ async def test_get_run_lineage_include_columns_with_depth(
994986
assert response.json() == {
995987
"relations": {
996988
"parents": run_parents_to_json(runs),
997-
"ancestors": [],
998989
"symlinks": [],
999990
"inputs": [
1000991
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -1150,7 +1141,6 @@ async def test_get_job_lineage_include_columns_with_depth(
11501141
assert response.json() == {
11511142
"relations": {
11521143
"parents": [],
1153-
"ancestors": [],
11541144
"symlinks": [],
11551145
"inputs": inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
11561146
"outputs": outputs_to_json(merge_io_by_jobs(outputs), granularity="JOB"),
@@ -1262,7 +1252,6 @@ async def test_get_dataset_lineage_with_granularity_dataset_and_column_lineage(
12621252
assert response.json() == {
12631253
"relations": {
12641254
"parents": [],
1265-
"ancestors": [],
12661255
"symlinks": [],
12671256
"outputs": [],
12681257
"inputs": sorted(
@@ -1395,7 +1384,6 @@ async def test_get_dataset_lineage_with_granularity_dataset_and_column_lineage_f
13951384
assert response.json() == {
13961385
"relations": {
13971386
"parents": [],
1398-
"ancestors": [],
13991387
"symlinks": [],
14001388
"outputs": [],
14011389
"inputs": sorted(

‎tests/test_server/test_lineage/test_dataset_lineage.py‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@ async def test_get_dataset_lineage_no_relations(
8383
assert response.json() == {
8484
"relations": {
8585
"parents": [],
86-
"ancestors": [],
8786
"symlinks": [],
8887
"inputs": [],
8988
"outputs": [],
@@ -141,7 +140,6 @@ async def test_get_dataset_lineage_with_granularity_run(
141140
assert response.json() == {
142141
"relations": {
143142
"parents": run_parents_to_json(runs),
144-
"ancestors": [],
145143
"symlinks": [],
146144
"inputs": [
147145
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -201,7 +199,6 @@ async def test_get_dataset_lineage_with_granularity_job(
201199
assert response.json() == {
202200
"relations": {
203201
"parents": [],
204-
"ancestors": [],
205202
"symlinks": [],
206203
"inputs": inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
207204
"outputs": outputs_to_json(merge_io_by_jobs(outputs), granularity="JOB"),
@@ -264,7 +261,6 @@ async def test_get_dataset_lineage_with_granularity_operation(
264261
assert response.json() == {
265262
"relations": {
266263
"parents": run_parents_to_json(runs) + operation_parents_to_json(operations),
267-
"ancestors": [],
268264
"symlinks": [],
269265
"inputs": [
270266
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -319,7 +315,6 @@ async def test_get_dataset_lineage_with_granularity_dataset(
319315
assert response.json() == {
320316
"relations": {
321317
"parents": [],
322-
"ancestors": [],
323318
"symlinks": [],
324319
"outputs": [],
325320
"inputs": sorted(
@@ -396,7 +391,6 @@ async def test_get_dataset_lineage_with_granularity_dataset_and_direction(
396391
assert response.json() == {
397392
"relations": {
398393
"parents": [],
399-
"ancestors": [],
400394
"symlinks": [],
401395
"outputs": [],
402396
"inputs": sorted(
@@ -465,7 +459,6 @@ async def test_get_dataset_lineage_with_granularity_dataset_and_depth(
465459
assert response.json() == {
466460
"relations": {
467461
"parents": [],
468-
"ancestors": [],
469462
"symlinks": [],
470463
"outputs": [],
471464
"inputs": sorted(
@@ -539,7 +532,6 @@ async def test_get_dataset_lineage_with_granularity_dataset_and_symlinks(
539532
assert response.json() == {
540533
"relations": {
541534
"parents": [],
542-
"ancestors": [],
543535
"symlinks": symlinks_to_json(dataset_symlinks),
544536
"outputs": [],
545537
"inputs": sorted(
@@ -615,7 +607,6 @@ async def test_get_dataset_lineage_with_granularity_dataset_and_until(
615607
assert response.json() == {
616608
"relations": {
617609
"parents": [],
618-
"ancestors": [],
619610
"symlinks": [],
620611
"outputs": [],
621612
"inputs": sorted(
@@ -691,7 +682,6 @@ async def test_get_dataset_lineage_with_direction_downstream(
691682
assert response.json() == {
692683
"relations": {
693684
"parents": run_parents_to_json(runs),
694-
"ancestors": [],
695685
"symlinks": [],
696686
"inputs": [
697687
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -750,7 +740,6 @@ async def test_get_dataset_lineage_with_direction_upstream(
750740
assert response.json() == {
751741
"relations": {
752742
"parents": run_parents_to_json(runs),
753-
"ancestors": [],
754743
"symlinks": [],
755744
"inputs": [],
756745
"outputs": [
@@ -817,7 +806,6 @@ async def test_get_dataset_lineage_with_until(
817806
assert response.json() == {
818807
"relations": {
819808
"parents": run_parents_to_json(runs),
820-
"ancestors": [],
821809
"symlinks": [],
822810
"inputs": [
823811
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -916,7 +904,6 @@ async def test_get_dataset_lineage_with_depth(
916904
assert response.json() == {
917905
"relations": {
918906
"parents": run_parents_to_json(runs),
919-
"ancestors": [],
920907
"symlinks": [],
921908
"inputs": [
922909
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -1008,7 +995,6 @@ async def test_get_dataset_lineage_with_depth_and_granularity_job(
1008995
assert response.json() == {
1009996
"relations": {
1010997
"parents": [],
1011-
"ancestors": [],
1012998
"symlinks": [],
1013999
"inputs": inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
10141000
"outputs": outputs_to_json(merge_io_by_jobs(outputs), granularity="JOB"),
@@ -1107,7 +1093,6 @@ async def test_get_dataset_lineage_with_depth_and_granularity_operation(
11071093
assert response.json() == {
11081094
"relations": {
11091095
"parents": run_parents_to_json(runs) + operation_parents_to_json(operations),
1110-
"ancestors": [],
11111096
"symlinks": [],
11121097
"inputs": [
11131098
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -1164,7 +1149,6 @@ async def test_get_dataset_lineage_with_depth_ignore_cycles(
11641149
assert response.json() == {
11651150
"relations": {
11661151
"parents": run_parents_to_json(runs),
1167-
"ancestors": [],
11681152
"symlinks": [],
11691153
"inputs": [
11701154
*inputs_to_json(merge_io_by_jobs(lineage.inputs), granularity="JOB"),
@@ -1249,7 +1233,6 @@ async def test_get_dataset_lineage_with_depth_ignore_unrelated_datasets(
12491233
assert response.json() == {
12501234
"relations": {
12511235
"parents": run_parents_to_json(runs),
1252-
"ancestors": [],
12531236
"symlinks": [],
12541237
"inputs": [
12551238
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -1329,7 +1312,6 @@ async def test_get_dataset_lineage_with_symlink(
13291312
assert response.json() == {
13301313
"relations": {
13311314
"parents": run_parents_to_json(runs),
1332-
"ancestors": [],
13331315
"symlinks": symlinks_to_json(dataset_symlinks),
13341316
"inputs": [
13351317
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -1410,7 +1392,6 @@ async def test_get_dataset_lineage_with_symlink_without_input_output(
14101392
assert response.json() == {
14111393
"relations": {
14121394
"parents": run_parents_to_json(runs),
1413-
"ancestors": [],
14141395
"symlinks": symlinks_to_json(dataset_symlinks),
14151396
"inputs": [
14161397
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -1495,7 +1476,6 @@ async def test_get_dataset_lineage_unmergeable_schema_and_output_type(
14951476
assert response.json() == {
14961477
"relations": {
14971478
"parents": run_parents_to_json(runs),
1498-
"ancestors": [],
14991479
"symlinks": [],
15001480
"inputs": [
15011481
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -1611,7 +1591,6 @@ async def test_get_dataset_lineage_empty_io_stats_and_schema(
16111591
assert response.json() == {
16121592
"relations": {
16131593
"parents": run_parents_to_json(runs),
1614-
"ancestors": [],
16151594
"symlinks": [],
16161595
"inputs": [
16171596
*inputs_to_json(merged_job_inputs, granularity="JOB"),
@@ -1680,7 +1659,6 @@ async def test_get_dataset_lineage_with_granularity_dataset_without_output_schem
16801659
assert response.json() == {
16811660
"relations": {
16821661
"parents": [],
1683-
"ancestors": [],
16841662
"symlinks": [],
16851663
"outputs": [],
16861664
"inputs": sorted(
@@ -1758,7 +1736,6 @@ async def test_get_dataset_lineage_with_granularity_dataset_ignore_self_referenc
17581736
assert response.json() == {
17591737
"relations": {
17601738
"parents": [],
1761-
"ancestors": [],
17621739
"symlinks": [],
17631740
"inputs": [],
17641741
"outputs": [],
@@ -1812,7 +1789,6 @@ async def test_get_dataset_lineage_with_granularity_dataset_ignore_not_connected
18121789
assert response.json() == {
18131790
"relations": {
18141791
"parents": [],
1815-
"ancestors": [],
18161792
"symlinks": [],
18171793
"inputs": [],
18181794
"outputs": [],
@@ -1892,7 +1868,6 @@ async def test_get_dataset_lineage_for_long_running_operations_with_granularity_
18921868
assert response.json() == {
18931869
"relations": {
18941870
"parents": run_parents_to_json(runs),
1895-
"ancestors": [],
18961871
"symlinks": [],
18971872
"inputs": [
18981873
*inputs_to_json(merge_io_by_jobs(inputs), granularity="JOB"),
@@ -1978,7 +1953,6 @@ async def test_get_dataset_lineage_for_long_running_operations_with_granularity_
19781953
assert response.json() == {
19791954
"relations": {
19801955
"parents": [],
1981-
"ancestors": [],
19821956
"symlinks": [],
19831957
"inputs": sorted(
19841958
[

‎tests/test_server/test_lineage/test_get_lineage_request_validators.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,6 @@ async def test_get_lineage_missing_id(
9090
assert response.json() == {
9191
"relations": {
9292
"parents": [],
93-
"ancestors": [],
9493
"symlinks": [],
9594
"inputs": [],
9695
"outputs": [],

0 commit comments

Comments
 (0)