Add Column Level Lineage support into HttpOpenLineageDispatcher

[cerveada]

Schema Dataset Facet

• https://openlineage.io/docs/spec/facets/dataset-facets/schema/
• json spec: https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json

Column Level Lineage Dataset Facet

• https://openlineage.io/docs/spec/facets/dataset-facets/column_lineage_facet/
• json spec: https://openlineage.io/spec/facets/1-2-0/ColumnLineageDatasetFacet.json

Desired output example

{
  "events": [
    {
      "eventType": "COMPLETE",                               //  Marquez only commits lineage on COMPLETE/FAILED event.
      "eventTime": "2025-02-14T15:50:27.462Z",
      "run": { "runId": "01950527-46c0-7384-a258-dda3ac77aba0", ... },         //  Ties inputs and outputs to a single job execution.
      "job": {.                                
        "namespace": "default",          // Identifier of the job.
        "name": "my_job",                  // Identifier of the job.
        ...
      },
      "inputs": [              // defines the source dataset
        {
          "namespace": "s3://211125613301-athenixlake-dev-bronze",
          "name": "opendoor_data/active",
          "facets": {                                                      
            "schema": {                  // This is the field-level info that was missing in the captured events. Without the input schema, the column lineage on the output has nothing to point back to, so we cannot extract column-level lineage.
              "fields": [
                { "name": "identifier", "type": "string" },
                { "name": "postalCodes", "type": "array", ... }
              ]
            },
            ...
          }
        }
      ],
      "outputs": [               // defines the output dataset
        {
          "namespace": "s3://211125613301-athenixlake-dev-gold",
          "name": "prestage/opendoor_markets",
          "facets": {
            "schema": {        //  The output columns. Needed so that each target field exists as a node to draw lineage into.
              "fields": [
                { "name": "identifier", "type": "string" },
                ...
              ]
            },
            "columnLineage": {                   // The key-part of column-level lineage. For each output field it lists the inputFields (which source dataset + which field) and the transformations describing how it was derived (here DIRECT/IDENTITY = copied straight through). This is what was empty in your events.
              "fields": {
                "identifier": {
                  "inputFields": [
                    {
                      "namespace": "s3://211125613301-athenixlake-dev-bronze",
                      "name": "opendoor_data/active",
                      "field": "identifier",
                      "transformations": [
                        { "type": "DIRECT", "subtype": "IDENTITY", "masking": false }
                      ]
                    }
                  ]
                },
                ...
              }
            },
            ...
          }
        }
      ]
    }
  ]
}