From 9bb9da5437d29a0d96d2419af829eebb4ebdf1ed Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Wed, 14 Dec 2022 19:45:21 +0530 Subject: [PATCH 1/4] ML Ingestion Spec Updation --- ... => raw_ml_obs_domain_criteria_index.json} | 12 ++ ...ain_index => raw_ml_obs_domain_index.json} | 12 ++ ...tus_index => raw_ml_obs_status_index.json} | 12 ++ ...> raw_ml_project_program_level_index.json} | 0 ...index => raw_ml_project_status_index.json} | 12 ++ .../templates/raw_sl_observation | 12 +- .../templates/raw_sl_observation_evidence | 4 - ...status => raw_sl_observation_status_index} | 60 ++++++ .../templates/raw_sl_project_index | 36 ++++ .../druid-ingestion/templates/raw_sl_survey | 5 + .../templates/raw_sl_survey_status_index.json | 185 ++++++++++++++++++ 11 files changed, 342 insertions(+), 8 deletions(-) rename ansible/roles/druid-ingestion/templates/{raw_ml_obs_domain_criteria_index => raw_ml_obs_domain_criteria_index.json} (91%) rename ansible/roles/druid-ingestion/templates/{raw_ml_obs_domain_index => raw_ml_obs_domain_index.json} (90%) rename ansible/roles/druid-ingestion/templates/{raw_ml_obs_status_index => raw_ml_obs_status_index.json} (90%) rename ansible/roles/druid-ingestion/templates/{raw_ml_project_program_level_index => raw_ml_project_program_level_index.json} (100%) rename ansible/roles/druid-ingestion/templates/{raw_ml_project_status_index => raw_ml_project_status_index.json} (90%) rename ansible/roles/druid-ingestion/templates/{raw_sl_observation_status => raw_sl_observation_status_index} (76%) create mode 100644 ansible/roles/druid-ingestion/templates/raw_sl_survey_status_index.json diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_criteria_index b/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_criteria_index.json similarity index 91% rename from ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_criteria_index rename to ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_criteria_index.json index fa3558a22e..b1258d1e7c 100644 --- a/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_criteria_index +++ b/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_criteria_index.json @@ -69,6 +69,14 @@ "type": "string", "name": "district_externalId" }, + { + "type":"string", + "name":"block_name" + }, + { + "type":"string", + "name":"block_externalId" + }, { "type": "string", "name": "organisation_name" @@ -125,6 +133,10 @@ "type": "long", "name": "unique_users" }, + { + "type":"long", + "name":"unique_solution" + }, { "type": "string", "name": "time_stamp" diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_index b/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_index.json similarity index 90% rename from ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_index rename to ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_index.json index 37eade976e..d41ebf4690 100644 --- a/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_index +++ b/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_index.json @@ -69,6 +69,14 @@ "type": "string", "name": "district_externalId" }, + { + "type":"string", + "name":"block_externalId" + }, + { + "type":"string", + "name":"organisation_name" + }, { "type": "string", "name": "organisation_name" @@ -113,6 +121,10 @@ "type": "long", "name": "unique_users" }, + { + "type":"long", + "name":"unique_solution" + }, { "type": "string", "name": "time_stamp" diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_obs_status_index b/ansible/roles/druid-ingestion/templates/raw_ml_obs_status_index.json similarity index 90% rename from ansible/roles/druid-ingestion/templates/raw_ml_obs_status_index rename to ansible/roles/druid-ingestion/templates/raw_ml_obs_status_index.json index 84f15cf77a..3800e27d20 100644 --- a/ansible/roles/druid-ingestion/templates/raw_ml_obs_status_index +++ b/ansible/roles/druid-ingestion/templates/raw_ml_obs_status_index.json @@ -73,6 +73,14 @@ "type": "string", "name": "district_externalId" }, + { + "type":"string", + "name":"block_name" + }, + { + "type":"string", + "name":"block_externalId" + }, { "type": "string", "name": "organisation_name" @@ -105,6 +113,10 @@ "type": "long", "name": "unique_users" }, + { + "type":"long", + "name":"unique_solution" + }, { "type": "string", "name": "time_stamp" diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index b/ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index.json similarity index 100% rename from ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index rename to ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index.json diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index b/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index.json similarity index 90% rename from ansible/roles/druid-ingestion/templates/raw_ml_project_status_index rename to ansible/roles/druid-ingestion/templates/raw_ml_project_status_index.json index 99f875c63c..e9403cb11a 100644 --- a/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index +++ b/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index.json @@ -73,6 +73,14 @@ "type": "string", "name": "district_externalId" }, + { + "type":"string", + "name":"block_name" + }, + { + "type":"string", + "name":"block_externalId" + }, { "type": "string", "name": "organisation_name" @@ -101,6 +109,10 @@ "type": "long", "name": "unique_users" }, + { + "type":"long", + "name":"unique_solution" + }, { "type": "long", "name": "no_of_imp_with_evidence" diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation b/ansible/roles/druid-ingestion/templates/raw_sl_observation index f2a0a7da72..3605ee2039 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_observation +++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation @@ -40,10 +40,6 @@ "type": "string", "name": "entityType" }, - { - "type": "string", - "name": "entityTypeId" - }, { "type": "string", "name": "observationId" @@ -471,6 +467,14 @@ { "type": "string", "name": "cluster_code" + }, + { + "type":"boolean", + "name":"isRubricDriven" + }, + { + "type":"boolean", + "name":"criteriaLevelReport" } ] } diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation_evidence b/ansible/roles/druid-ingestion/templates/raw_sl_observation_evidence index 9e2122ebc3..7f97d51f11 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_observation_evidence +++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation_evidence @@ -32,10 +32,6 @@ "type": "string", "name": "entityName" }, - { - "type": "string", - "name": "entityTypeId" - }, { "type": "string", "name": "entityType" diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation_status b/ansible/roles/druid-ingestion/templates/raw_sl_observation_status_index similarity index 76% rename from ansible/roles/druid-ingestion/templates/raw_sl_observation_status rename to ansible/roles/druid-ingestion/templates/raw_sl_observation_status_index index 2d49030ca7..6857ad9281 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_observation_status +++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation_status_index @@ -200,6 +200,66 @@ { "type": "string", "name": "user_type" + }, + { + "type": "string", + "name": "observed_school_name" + }, + { + "type": "string", + "name": "observed_school_id" + }, + { + "type": "string", + "name": "observed_school_code" + }, + { + "type": "string", + "name": "observed_state_name" + }, + { + "type": "string", + "name": "observed_state_id" + }, + { + "type": "string", + "name": "observed_state_code" + }, + { + "type": "string", + "name": "observed_district_name" + }, + { + "type": "string", + "name": "observed_district_id" + }, + { + "type": "string", + "name": "observed_district_code" + }, + { + "type": "string", + "name": "observed_block_name" + }, + { + "type": "string", + "name": "observed_block_id" + }, + { + "type": "string", + "name": "observed_block_code" + }, + { + "type": "string", + "name": "observed_cluster_name" + }, + { + "type": "string", + "name": "observed_cluster_id" + }, + { + "type": "string", + "name": "observed_cluster_code" } ] }, diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_project_index b/ansible/roles/druid-ingestion/templates/raw_sl_project_index index 1e6881d72d..3458aaa7f8 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_project_index +++ b/ansible/roles/druid-ingestion/templates/raw_sl_project_index @@ -288,6 +288,42 @@ { "type": "string", "name": "user_type" + }, + { + "type": "string", + "name": "certificate_id" + }, + { + "type": "string", + "name": "certificate_status" + }, + { + "type": "string", + "name": "certificate_issued_on" + }, + { + "type": "string", + "name": "certificate_status_customised" + }, + { + "type": "string", + "name": "certificate_template_url" + }, + { + "type":"long", + "name":"task_count" + }, + { + "type":"long", + "name":"task_evidence_count" + }, + { + "type":"long", + "name":"project_evidence_count" + }, + { + "type":"long", + "name":"task_sequence" } ] }, diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_survey b/ansible/roles/druid-ingestion/templates/raw_sl_survey index 18719f9a7f..834a0034be 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_survey +++ b/ansible/roles/druid-ingestion/templates/raw_sl_survey @@ -283,7 +283,12 @@ { "type": "string", "name": "organisation_id" + }, + { + "type":"string", + "name":"evidences" } + ] } } diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_survey_status_index.json b/ansible/roles/druid-ingestion/templates/raw_sl_survey_status_index.json new file mode 100644 index 0000000000..9eb45ee07e --- /dev/null +++ b/ansible/roles/druid-ingestion/templates/raw_sl_survey_status_index.json @@ -0,0 +1,185 @@ +{ + "type": "index", + "spec": { + "ioConfig": { + "type": "index", + "inputSource": { + "type": "azure", + "uris": [ + "azure://telemetry-data-store/survey/status/sl_survey_status.json" + ] + }, + "inputFormat": { + "type": "json" + } + }, + "tuningConfig": { + "type": "index", + "partitionsSpec": { + "type": "dynamic" + } + }, + "dataSchema": { + "dataSource": "ml-survey-status", + "granularitySpec": { + "type": "uniform", + "queryGranularity": "DAY", + "rollup": true, + "segmentGranularity": "DAY" + }, + "timestampSpec": { + "column": "updatedAt", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "survey_submission_id" + }, + { + "type": "string", + "name": "submission_status" + }, + { + "type": "string", + "name": "user_id" + }, + { + "type": "string", + "name": "user_sub_type" + }, + { + "type": "string", + "name": "user_type" + }, + { + "type": "string", + "name": "state_externalId" + }, + { + "type": "string", + "name": "block_externalId" + }, + { + "type": "string", + "name": "district_externalId" + }, + { + "type": "string", + "name": "cluster_externalId" + }, + { + "type": "string", + "name": "school_externalId" + }, + { + "type": "string", + "name": "state_name" + }, + { + "type": "string", + "name": "block_name" + }, + { + "type": "string", + "name": "district_name" + }, + { + "type": "string", + "name": "cluster_name" + }, + { + "type": "string", + "name": "school_name" + }, + { + "type": "string", + "name": "board_name" + }, + { + "type": "string", + "name": "organisation_id" + }, + { + "type": "string", + "name": "organisation_name" + }, + { + "type": "string", + "name": "program_externalId" + }, + { + "type": "string", + "name": "program_id" + }, + { + "type": "string", + "name": "program_name" + }, + { + "type": "string", + "name": "survey_name" + }, + { + "type": "string", + "name": "survey_id" + }, + { + "type": "string", + "name": "survey_externalId" + }, + { + "type": "string", + "name": "created_date" + }, + { + "type": "string", + "name": "submission_date" + }, + { + "type": "string", + "name": "updatedAt" + }, + { + "type": "string", + "name": "parent_channel" + }, + { + "type": "string", + "name": "solution_name" + }, + { + "type": "string", + "name": "solution_id" + }, + { + "type": "string", + "name": "private_program" + }, + { + "type": "string", + "name": "state_code" + }, + { + "type": "string", + "name": "school_code" + }, + { + "type": "string", + "name": "district_code" + }, + { + "type": "string", + "name": "block_code" + }, + { + "type": "string", + "name": "cluster_code" + } + ] + }, + "metricsSpec": [] + } + } +} \ No newline at end of file From 19b838907511b0585461f8e81e48b812bc6d20a8 Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Thu, 15 Dec 2022 16:24:25 +0530 Subject: [PATCH 2/4] Observation Ingestion Spec Correction --- ansible/roles/druid-ingestion/templates/raw_sl_observation | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation b/ansible/roles/druid-ingestion/templates/raw_sl_observation index 3605ee2039..c7cc4807c8 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_observation +++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation @@ -471,10 +471,6 @@ { "type":"boolean", "name":"isRubricDriven" - }, - { - "type":"boolean", - "name":"criteriaLevelReport" } ] } From 0f09e59d4418fc2dbb0acc7e560097a6bc7f6c76 Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Thu, 15 Dec 2022 18:03:27 +0530 Subject: [PATCH 3/4] Observation Status Ingestion Spec Updation --- .../templates/raw_sl_observation_status_index | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation_status_index b/ansible/roles/druid-ingestion/templates/raw_sl_observation_status_index index 6857ad9281..03a48ae5c1 100644 --- a/ansible/roles/druid-ingestion/templates/raw_sl_observation_status_index +++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation_status_index @@ -260,6 +260,14 @@ { "type": "string", "name": "observed_cluster_code" + }, + { + "name": "criteriaLevelReport", + "type": "boolean" + }, + { + "name": "isRubricDriven", + "type": "boolean" } ] }, From 8daba9ef534c6477621e01fb1b2b506caa301341 Mon Sep 17 00:00:00 2001 From: Shakthieshwari Date: Fri, 16 Dec 2022 10:32:44 +0530 Subject: [PATCH 4/4] Projects Ingestion Spec Updation --- .../templates/raw_ml_project_program_level_index.json | 4 ++++ .../templates/raw_ml_project_status_index.json | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index.json b/ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index.json index bb214cf72b..0dd29016be 100644 --- a/ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index.json +++ b/ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index.json @@ -84,6 +84,10 @@ { "type": "string", "name": "time_stamp" + }, + { + "type":"long", + "name":"no_of_certificate_issued" } ] }, diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index.json b/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index.json index e9403cb11a..bad246b2fc 100644 --- a/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index.json +++ b/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index.json @@ -120,7 +120,11 @@ { "type": "string", "name": "time_stamp" - } + }, + { + "type":"long", + "name":"no_of_certificate_issued" + } ] }, "metricsSpec": []