diff --git a/meta_update_middleware/db-utils.go b/meta_update_middleware/db-utils.go index 462d3e4c..067d3a2a 100644 --- a/meta_update_middleware/db-utils.go +++ b/meta_update_middleware/db-utils.go @@ -22,7 +22,7 @@ func init() { log.Println("db-utils:init()") } -func getDbConnection(cred Credentials) (conn CbConnection) { +func getDbConnection(cred Credentials) (conn CbConnection, err error) { log.Println("getDbConnection()") conn = CbConnection{} @@ -41,8 +41,8 @@ func getDbConnection(cred Credentials) (conn CbConnection) { cluster, err := gocb.Connect(connectionString, options) if err != nil { - log.Fatal(err) - return + err = fmt.Errorf("failed to connect to cluster: %w", err) + return conn, err } conn.Cluster = cluster @@ -54,36 +54,42 @@ func getDbConnection(cred Credentials) (conn CbConnection) { err = conn.Bucket.WaitUntilReady(5*time.Second, nil) if err != nil { - log.Fatal(err) - return + return conn, err } conn.Scope = conn.Bucket.Scope(cred.Cb_scope) - return conn + return conn, nil } -func queryWithSQLFile(scope *gocb.Scope, file string) (jsonOut []string) { +func queryWithSQLFile(scope *gocb.Scope, file string) (jsonOut []string, err error) { fileContent, err := os.ReadFile(file) if err != nil { - log.Fatal(err) + log.Println("Error reading file:", err) + return nil, err } // Convert []byte to string text := string(fileContent) fmt.Println(text) - - return queryWithSQLStringSA(scope, text) + result, err := queryWithSQLStringSA(scope, text) + if err != nil { + return nil, err + } + return result, nil } -func queryWithSQLStringSA(scope *gocb.Scope, text string) (rv []string) { +func queryWithSQLStringSA(scope *gocb.Scope, text string) (rv []string, err error) { log.Println("queryWithSQLStringSA(\n" + text + "\n)") queryResult, err := scope.Query( fmt.Sprintf(text), - &gocb.QueryOptions{Adhoc: true}, + &gocb.QueryOptions{ + Adhoc: true, + Timeout: 300 * time.Second, // Set an explicit timeout + }, ) if err != nil { - log.Fatal(err) + return nil, err } // Interfaces for handling streaming return values @@ -95,23 +101,23 @@ func queryWithSQLStringSA(scope *gocb.Scope, text string) (rv []string) { var row interface{} err := queryResult.Row(&row) if err != nil { - log.Fatal(err) + return nil, err } retValues = append(retValues, row.(string)) } - return retValues + return retValues, nil } -func queryWithSQLStringFA(scope *gocb.Scope, text string) (rv []float64) { +func queryWithSQLStringFA(scope *gocb.Scope, text string) (rv []float64, err error) { log.Println("queryWithSQLStringFA(\n" + text + "\n)") queryResult, err := scope.Query( fmt.Sprintf(text), - &gocb.QueryOptions{Adhoc: true}, + &gocb.QueryOptions{Adhoc: true, Timeout: 300 * time.Second}, ) if err != nil { - log.Fatal(err) + return nil, err } retValues := make([]float64, 0) @@ -122,23 +128,23 @@ func queryWithSQLStringFA(scope *gocb.Scope, text string) (rv []float64) { var row interface{} err := queryResult.Row(&row) if err != nil { - log.Fatal(err) + return nil, err } retValues = append(retValues, row.(float64)) } - return retValues + return retValues, nil } -func queryWithSQLStringIA(scope *gocb.Scope, text string) (rv []int) { +func queryWithSQLStringIA(scope *gocb.Scope, text string) (rv []int, err error) { log.Println("queryWithSQLStringFA(\n" + text + "\n)") queryResult, err := scope.Query( fmt.Sprintf(text), - &gocb.QueryOptions{Adhoc: true}, + &gocb.QueryOptions{Adhoc: true, Timeout: 300 * time.Second}, ) if err != nil { - log.Fatal(err) + return nil, err } retValues := make([]int, 0) @@ -149,28 +155,28 @@ func queryWithSQLStringIA(scope *gocb.Scope, text string) (rv []int) { var row interface{} err := queryResult.Row(&row) if err != nil { - log.Fatal(err) + return nil, err } - switch row.(type) { + switch v := row.(type) { case float64: - retValues = append(retValues, int(row.(float64))) + retValues = append(retValues, int(v)) case int: - retValues = append(retValues, row.(int)) + retValues = append(retValues, v) } } - return retValues + return retValues, nil } -func queryWithSQLStringMAP(scope *gocb.Scope, text string) (jsonOut []interface{}) { +func queryWithSQLStringMAP(scope *gocb.Scope, text string) (jsonOut []interface{}, err error) { log.Println("queryWithSQLStringMAP(\n" + text + "\n)") queryResult, err := scope.Query( fmt.Sprintf(text), - &gocb.QueryOptions{Adhoc: true}, + &gocb.QueryOptions{Adhoc: true, Timeout: 300 * time.Second}, ) if err != nil { - log.Fatal(err) + return nil, err } rows := make([]interface{}, 0) @@ -179,18 +185,18 @@ func queryWithSQLStringMAP(scope *gocb.Scope, text string) (jsonOut []interface{ var row interface{} err := queryResult.Row(&row) if err != nil { - log.Fatal(err) + return nil, err } m := row.(map[string]interface{}) rows = append(rows, m) } - return rows + return rows, nil } -func queryWithSQLFileJustPrint(scope *gocb.Scope, file string) { +func queryWithSQLFileJustPrint(scope *gocb.Scope, file string) error { fileContent, err := os.ReadFile(file) if err != nil { - log.Fatal(err) + return err } // Convert []byte to string @@ -199,22 +205,22 @@ func queryWithSQLFileJustPrint(scope *gocb.Scope, file string) { queryResult, err := scope.Query( fmt.Sprintf(text), - &gocb.QueryOptions{Adhoc: true}, + &gocb.QueryOptions{Adhoc: true, Timeout: 300 * time.Second}, ) if err != nil { - log.Fatal(err) - } else { - printQueryResult(queryResult) + return err } + return printQueryResult(queryResult) } -func printQueryResult(queryResult *gocb.QueryResult) { +func printQueryResult(queryResult *gocb.QueryResult) error { for queryResult.Next() { var result interface{} err := queryResult.Row(&result) if err != nil { - log.Fatal(err) + return err } fmt.Println(result) } + return nil } diff --git a/meta_update_middleware/meta-update.go b/meta_update_middleware/meta-update.go index f4367b9c..ccc41d21 100644 --- a/meta_update_middleware/meta-update.go +++ b/meta_update_middleware/meta-update.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "flag" - "fmt" "log" "os" "time" @@ -63,8 +62,6 @@ func main() { log.Println("meta-update, settings file:" + settingsFilePath + ",credentials file:" + credentialsFilePath + ",app:[all apps in settings file]") } - conf := ConfigJSON{} - conf, err := parseConfig(settingsFilePath) if err != nil { log.Fatal("Unable to parse config") @@ -73,7 +70,10 @@ func main() { credentials := getCredentials(credentialsFilePath) - conn := getDbConnection(credentials) + conn, err := getDbConnection(credentials) + if err != nil { + log.Fatalf("Failed to connect to database: %v", err) + } //testGetSingleCTC(conn) //testGetCTCCount(connSrc) @@ -89,20 +89,28 @@ func main() { updateMedataForAppDocType(conn, conf.Metadata[ds].Name, conf.Metadata[ds].App, conf.Metadata[ds].DocType[dt], conf.Metadata[ds].SubDocType) } } - log.Println(fmt.Sprintf("\tmeta update finished in %v", time.Since(start))) + log.Printf("\tmeta update finished in %v", time.Since(start)) } func updateMedataForAppDocType(conn CbConnection, name string, app string, doctype string, subDocType string) { log.Println("updateMedataForAppDocType(" + name + "," + doctype + ")") // get needed models - models := getModels(conn, name, app, doctype, subDocType) + models, err := getModels(conn, name, app, doctype, subDocType) + if err != nil { + log.Printf("Error getting models: %v", err) + return + } log.Println("models:") printStringArray(models) // get models having metadata but no data (remove metadata for these) // (note 'like %' is changed to 'like %25') - models_with_metatada_but_no_data := getModelsNoData(conn, name, app, doctype, subDocType) + models_with_metatada_but_no_data, err := getModelsNoData(conn, name, app, doctype, subDocType) + if err != nil { + log.Printf("Error getting models with metadata but no data: %v", err) + return + } log.Println("models_with_metatada_but_no_data:") printStringArray(models_with_metatada_but_no_data) @@ -132,20 +140,50 @@ func updateMedataForAppDocType(conn CbConnection, name string, app string, docty metadata.Updated = 0 for i, m := range models { + var err error + var thresholds []string model := Model{Name: m} - thresholds := getDistinctThresholds(conn, name, app, doctype, subDocType, m) + thresholds, err = getDistinctThresholds(conn, name, app, doctype, subDocType, m) + if err != nil { + log.Printf("Error getting distinct thresholds: %v", err) + continue + } log.Println(thresholds) - fcstLen := getDistinctFcstLen(conn, name, app, doctype, subDocType, m) + fcstLen, err := getDistinctFcstLen(conn, name, app, doctype, subDocType, m) + if err != nil { + log.Printf("Error getting distinct forecast lengths: %v", err) + continue + } log.Println(fcstLen) - region := getDistinctRegion(conn, name, app, doctype, subDocType, m) + region, err := getDistinctRegion(conn, name, app, doctype, subDocType, m) + if err != nil { + log.Printf("Error getting distinct region: %v", err) + continue + } log.Println(region) - displayText := getDistinctDisplayText(conn, name, app, doctype, subDocType, m) + displayText, err := getDistinctDisplayText(conn, name, app, doctype, subDocType, m) + if err != nil { + log.Printf("Error getting distinct display text: %v", err) + continue + } log.Println(displayText) - displayCategory := getDistinctDisplayCategory(conn, name, app, doctype, subDocType, m) + displayCategory, err := getDistinctDisplayCategory(conn, name, app, doctype, subDocType, m) + if err != nil { + log.Printf("Error getting distinct display category: %v", err) + continue + } log.Println(displayCategory) - displayOrder := getDistinctDisplayOrder(conn, name, app, doctype, subDocType, m, i) + displayOrder, err := getDistinctDisplayOrder(conn, name, app, doctype, subDocType, m, i) + if err != nil { + log.Printf("Error getting distinct display order: %v", err) + continue + } log.Println(displayOrder) - minMaxCountFloor := getMinMaxCountFloor(conn, name, app, doctype, subDocType, m) + minMaxCountFloor, err := getMinMaxCountFloor(conn, name, app, doctype, subDocType, m) + if err != nil { + log.Printf("Error getting min/max/count/floor: %v", err) + continue + } log.Println(jsonPrettyPrintStruct(minMaxCountFloor[0].(map[string]interface{}))) // ./sqls/getDistinctThresholds.sql returns list of variables for SUMS DocType, like in Surface diff --git a/meta_update_middleware/template-queries.go b/meta_update_middleware/template-queries.go index cfa2c6d7..c9b12b2b 100644 --- a/meta_update_middleware/template-queries.go +++ b/meta_update_middleware/template-queries.go @@ -1,14 +1,12 @@ package main import ( - "fmt" + "github.com/couchbase/gocb/v2" "log" "os" "strconv" "strings" "time" - - "github.com/couchbase/gocb/v2" ) // init runs before main() is evaluated @@ -16,51 +14,57 @@ func init() { log.Println("template-queries:init()") } -func getModels(conn CbConnection, dataset string, app string, doctype string, subDocType string) (jsonOut []string) { +func getModels(conn CbConnection, dataset string, app string, doctype string, subDocType string) (jsonOut []string, err error) { log.Println("getModels(" + dataset + "," + app + "," + doctype + "," + subDocType + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getModels.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplGetModelsSQL := string(fileContent) tmplGetModelsSQL = strings.Replace(tmplGetModelsSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) tmplGetModelsSQL = strings.Replace(tmplGetModelsSQL, "{{vxDOCTYPE}}", doctype, -1) tmplGetModelsSQL = strings.Replace(tmplGetModelsSQL, "{{vxSUBDOCTYPE}}", subDocType, -1) - models_requiring_metadata := queryWithSQLStringSA(conn.Scope, tmplGetModelsSQL) + models_requiring_metadata, err := queryWithSQLStringSA(conn.Scope, tmplGetModelsSQL) + if err != nil { + return nil, err + } - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return models_requiring_metadata + log.Printf("\tin %v", time.Since(start)) + return models_requiring_metadata, nil } -func getModelsNoData(conn CbConnection, dataset string, app string, doctype string, subDocType string) (jsonOut []string) { +func getModelsNoData(conn CbConnection, dataset string, app string, doctype string, subDocType string) (jsonOut []string, err error) { log.Println("getModelsNoData(" + dataset + "," + app + "," + doctype + "," + subDocType + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getModelsNoData.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplgetModelsNoDataSQL := string(fileContent) tmplgetModelsNoDataSQL = strings.Replace(tmplgetModelsNoDataSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) tmplgetModelsNoDataSQL = strings.Replace(tmplgetModelsNoDataSQL, "{{vxDOCTYPE}}", doctype, -1) tmplgetModelsNoDataSQL = strings.Replace(tmplgetModelsNoDataSQL, "{{vxAPP}}", app, -1) tmplgetModelsNoDataSQL = strings.Replace(tmplgetModelsNoDataSQL, "{{vxSUBDOCTYPE}}", doctype, -1) - models_with_metatada_but_no_data := queryWithSQLStringSA(conn.Scope, tmplgetModelsNoDataSQL) + models_with_metatada_but_no_data, err := queryWithSQLStringSA(conn.Scope, tmplgetModelsNoDataSQL) + if err != nil { + return nil, err + } - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return models_with_metatada_but_no_data + log.Printf("\tin %v", time.Since(start)) + return models_with_metatada_but_no_data, nil } -func removeMetadataForModelsWithNoData(conn CbConnection, dataset string, app string, doctype string, subDocType string, models_with_metatada_but_no_data []string) { +func removeMetadataForModelsWithNoData(conn CbConnection, dataset string, app string, doctype string, subDocType string, models_with_metatada_but_no_data []string) (err error) { log.Println("removeMetadataForModelsWithNoData(" + dataset + "," + app + "," + doctype + "," + subDocType + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/deleteModelMetadata.sql") if err != nil { - log.Fatal(err) + return err } tmplDeleteModelMetadataSQL := string(fileContent) tmplDeleteModelMetadataSQL = strings.Replace(tmplDeleteModelMetadataSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) @@ -74,38 +78,42 @@ func removeMetadataForModelsWithNoData(conn CbConnection, dataset string, app st &gocb.QueryOptions{Adhoc: true}, ) if err != nil { - log.Fatal(err) + return err } else { printQueryResult(queryResult) } } - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) + log.Printf("\tin %v", time.Since(start)) + return nil } -func getModelsWithExistingMetadata(conn CbConnection, dataset string, app string, doctype string, subDocType string) (jsonOut []string) { +func getModelsWithExistingMetadata(conn CbConnection, dataset string, app string, doctype string, subDocType string) (jsonOut []string, err error) { log.Println("getModelsWithExistingMetadata(" + dataset + "," + app + "," + doctype + "," + subDocType + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getModelsWithMetadata.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplgetModelsWithMetadataSQL := string(fileContent) tmplgetModelsWithMetadataSQL = strings.Replace(tmplgetModelsWithMetadataSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, 1) tmplgetModelsWithMetadataSQL = strings.Replace(tmplgetModelsWithMetadataSQL, "{{vxAPP}}", app, -1) - models_with_existing_metadata := queryWithSQLStringSA(conn.Scope, tmplgetModelsWithMetadataSQL) - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return models_with_existing_metadata + models_with_existing_metadata, err := queryWithSQLStringSA(conn.Scope, tmplgetModelsWithMetadataSQL) + if err != nil { + return nil, err + } + log.Printf("\tin %v", time.Since(start)) + return models_with_existing_metadata, nil } -func initializeMetadataForModel(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) { +func initializeMetadataForModel(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (err error) { log.Println("initializeMetadataForModel(" + dataset + "," + app + "," + doctype + "," + subDocType + "," + model + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/initializeMetadata.sql") if err != nil { - log.Fatal(err) + return err } tmplInitializeMetadataSQL := string(fileContent) tmplInitializeMetadataSQL = strings.Replace(tmplInitializeMetadataSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) @@ -116,20 +124,21 @@ func initializeMetadataForModel(conn CbConnection, dataset string, app string, d tmplInitializeMetadataSQL, &gocb.QueryOptions{Adhoc: true}, ) if err != nil { - log.Fatal(err) + return err } else { printQueryResult(queryResult) } - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) + log.Printf("\tin %v", time.Since(start)) + return nil } -func getDistinctThresholds(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []string) { +func getDistinctThresholds(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []string, err error) { log.Println("getDistinctThresholds(" + dataset + "," + app + "," + doctype + "," + subDocType + "," + model + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getDistinctThresholds.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplSQL := string(fileContent) tmplSQL = strings.Replace(tmplSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) @@ -137,7 +146,10 @@ func getDistinctThresholds(conn CbConnection, dataset string, app string, doctyp tmplSQL = strings.Replace(tmplSQL, "{{vxSUBDOCTYPE}}", subDocType, -1) tmplSQL = strings.Replace(tmplSQL, "{{vxMODEL}}", model, -1) - result := queryWithSQLStringMAP(conn.Scope, tmplSQL) + result, err := queryWithSQLStringMAP(conn.Scope, tmplSQL) + if err != nil { + return nil, err + } m := result[0].(map[string]interface{}) if len(result) > 1 { @@ -163,17 +175,17 @@ func getDistinctThresholds(conn CbConnection, dataset string, app string, doctyp } */ - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return tarr + log.Printf("\tin %v", time.Since(start)) + return tarr, err } -func getDistinctFcstLen(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []int) { +func getDistinctFcstLen(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []int, err error) { log.Println("getDistinctFcstLen(" + dataset + "," + app + "," + doctype + "," + subDocType + "," + model + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getDistinctFcstLen.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplSQL := string(fileContent) tmplSQL = strings.Replace(tmplSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) @@ -181,18 +193,21 @@ func getDistinctFcstLen(conn CbConnection, dataset string, app string, doctype s tmplSQL = strings.Replace(tmplSQL, "{{vxSUBDOCTYPE}}", subDocType, -1) tmplSQL = strings.Replace(tmplSQL, "{{vxMODEL}}", model, -1) - result := queryWithSQLStringIA(conn.Scope, tmplSQL) - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return result + result, err := queryWithSQLStringIA(conn.Scope, tmplSQL) + if err != nil { + return nil, err + } + log.Printf("\tin %v", time.Since(start)) + return result, nil } -func getDistinctRegion(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []string) { +func getDistinctRegion(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []string, err error) { log.Println("getDistinctRegion(" + dataset + "," + app + "," + doctype + "," + subDocType + "," + model + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getDistinctRegion.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplSQL := string(fileContent) tmplSQL = strings.Replace(tmplSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) @@ -200,18 +215,21 @@ func getDistinctRegion(conn CbConnection, dataset string, app string, doctype st tmplSQL = strings.Replace(tmplSQL, "{{vxSUBDOCTYPE}}", subDocType, -1) tmplSQL = strings.Replace(tmplSQL, "{{vxMODEL}}", model, -1) - result := queryWithSQLStringSA(conn.Scope, tmplSQL) - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return result + result, err := queryWithSQLStringSA(conn.Scope, tmplSQL) + if err != nil { + return nil, err + } + log.Printf("\tin %v", time.Since(start)) + return result, nil } -func getDistinctDisplayText(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []string) { +func getDistinctDisplayText(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []string, err error) { log.Println("getDistinctDisplayText(" + dataset + "," + app + "," + doctype + "," + subDocType + "," + model + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getDistinctDisplayText.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplSQL := string(fileContent) tmplSQL = strings.Replace(tmplSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) @@ -219,18 +237,21 @@ func getDistinctDisplayText(conn CbConnection, dataset string, app string, docty tmplSQL = strings.Replace(tmplSQL, "{{vxSUBDOCTYPE}}", subDocType, -1) tmplSQL = strings.Replace(tmplSQL, "{{vxMODEL}}", model, -1) - result := queryWithSQLStringSA(conn.Scope, tmplSQL) - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return result + result, err := queryWithSQLStringSA(conn.Scope, tmplSQL) + if err != nil { + return nil, err + } + log.Printf("\tin %v", time.Since(start)) + return result, nil } -func getDistinctDisplayCategory(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []int) { +func getDistinctDisplayCategory(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (rv []int, err error) { log.Println("getDistinctDisplayCategory(" + dataset + "," + app + "," + doctype + "," + subDocType + "," + model + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getDistinctDisplayCategory.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplSQL := string(fileContent) tmplSQL = strings.Replace(tmplSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) @@ -238,18 +259,21 @@ func getDistinctDisplayCategory(conn CbConnection, dataset string, app string, d tmplSQL = strings.Replace(tmplSQL, "{{vxSUBDOCTYPE}}", subDocType, -1) tmplSQL = strings.Replace(tmplSQL, "{{vxMODEL}}", model, -1) - result := queryWithSQLStringIA(conn.Scope, tmplSQL) - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return result + result, err := queryWithSQLStringIA(conn.Scope, tmplSQL) + if err != nil { + return nil, err + } + log.Printf("\tin %v", time.Since(start)) + return result, nil } -func getDistinctDisplayOrder(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string, mindx int) (rv []int) { +func getDistinctDisplayOrder(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string, mindx int) (rv []int, err error) { log.Println("getDistinctDisplayOrder(" + dataset + "," + app + "," + doctype + "," + subDocType + "," + model + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getDistinctDisplayOrder.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplSQL := string(fileContent) tmplSQL = strings.Replace(tmplSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) @@ -258,18 +282,21 @@ func getDistinctDisplayOrder(conn CbConnection, dataset string, app string, doct tmplSQL = strings.Replace(tmplSQL, "{{vxMODEL}}", model, -1) tmplSQL = strings.Replace(tmplSQL, "{{mindx}}", strconv.Itoa(mindx), -1) - result := queryWithSQLStringIA(conn.Scope, tmplSQL) - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return result + result, err := queryWithSQLStringIA(conn.Scope, tmplSQL) + if err != nil { + return nil, err + } + log.Printf("\tin %v", time.Since(start)) + return result, nil } -func getMinMaxCountFloor(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (jsonOut []interface{}) { +func getMinMaxCountFloor(conn CbConnection, dataset string, app string, doctype string, subDocType string, model string) (jsonOut []interface{}, err error) { log.Println("getMinMaxCountFloor(" + dataset + "," + app + "," + doctype + "," + subDocType + "," + model + ")") start := time.Now() fileContent, err := os.ReadFile("sqls/getMinMaxCountFloor.sql") if err != nil { - log.Fatal(err) + return nil, err } tmplSQL := string(fileContent) tmplSQL = strings.Replace(tmplSQL, "{{vxDBTARGET}}", conn.vxDBTARGET, -1) @@ -277,7 +304,10 @@ func getMinMaxCountFloor(conn CbConnection, dataset string, app string, doctype tmplSQL = strings.Replace(tmplSQL, "{{vxSUBDOCTYPE}}", subDocType, -1) tmplSQL = strings.Replace(tmplSQL, "{{vxMODEL}}", model, -1) - result := queryWithSQLStringMAP(conn.Scope, tmplSQL) - log.Println(fmt.Sprintf("\tin %v", time.Since(start))) - return result + result, err := queryWithSQLStringMAP(conn.Scope, tmplSQL) + if err != nil { + return nil, err + } + log.Printf("\tin %v", time.Since(start)) + return result, nil } diff --git a/scripts/VXingest_utilities/run_job.sh b/scripts/VXingest_utilities/run_job.sh new file mode 100644 index 00000000..c7686a3d --- /dev/null +++ b/scripts/VXingest_utilities/run_job.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# ------------------------------------------------------------------------------ +# Script: run_job.sh +# Purpose: Submits and processes one or more VxIngest jobs by job ID. +# For each job ID, this script: +# - Validates the job ID format +# - Submits the job using Docker Compose +# - Runs an import script to process job documents +# - Cleans up small/no-op log files +# After all jobs, updates metadata using a Go program. +# Usage: +# ./run_job.sh [ ...] +# Each job_id must start with 'JS:'. +# ------------------------------------------------------------------------------ + +# Exit on error, unset variable, or failed pipe +set -euo pipefail + +# Check for at least one argument (job ID list) +if [ "$#" -lt 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Collect job IDs from arguments +job_ids=( "$@" ) + +for job_id in "${job_ids[@]}"; do + echo "Processing job_id: $job_id" + # Validate job_id format (must start with 'JS:') + if [[ ! "$job_id" =~ ^JS:.* ]]; then + echo "Error: job_id must start with 'JS:' $job_id" + exit 1 + fi + + echo "Submitting job with ID: $job_id" + # Submits the job using Docker Compose. Environment variables 'data' and 'public' are set for the container. + docker compose run --rm \ + -e data=/data-ingest/data \ + -e public=/public \ + ingest \ + -c /run/secrets/CREDENTIALS_FILE \ + -o /opt/data/outdir \ + -l /opt/data/logs \ + -m /opt/data/common/job_metrics \ + -x /opt/data/xfer \ + -j "$job_id" + + echo "Importing job documents for job ID: $job_id" + # Runs the import script to process job documents for this job ID. + cd /home/amb-verif/VxIngest + /home/amb-verif/VxIngest/scripts/VXingest_utilities/run-import.sh \ + -c /home/amb-verif/credentials \ + -l /data-ingest/data/xfer/ \ + -m /data-ingest/data/common/job_metrics \ + -t /data-ingest/data/temp_tar \ + > "/home/amb-verif/logs/cron-import-$(date +%s).out" 2>&1 + + # Clean up small/no-op log files (less than 100 bytes and containing 'No such file') + find ~/logs -type f -name 'cron-import-*.out' -size -100c -exec grep -l "No such file" {} \; -delete +done + +echo "add this retro to the metadata" + + +echo "Updating the metadata" +# After all jobs, update the metadata using the Go program in meta_update_middleware. +cd /home/amb-verif/VxIngest/meta_update_middleware +go run .