Skip to content

572 meta update middleware is not running to completeion #573

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
randytpierce wants to merge 2 commits into
base: main
Choose a base branch
from
Open

572 meta update middleware is not running to completeion #573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
127d4d6
increased db timeout and reworked error handling. Also added a run_jo…
randytpierce Dec 11, 2025
11fd79f
Merge branch '570-file_pattern-is-not-processing-correctly' into 572-…
randytpierce Dec 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 47 additions & 41 deletions meta_update_middleware/db-utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func init() {
log.Println("db-utils:init()")
}

func getDbConnection(cred Credentials) (conn CbConnection) {
func getDbConnection(cred Credentials) (conn CbConnection, err error) {
log.Println("getDbConnection()")

conn = CbConnection{}
Expand All @@ -41,8 +41,8 @@ func getDbConnection(cred Credentials) (conn CbConnection) {

cluster, err := gocb.Connect(connectionString, options)
if err != nil {
log.Fatal(err)
return
err = fmt.Errorf("failed to connect to cluster: %w", err)
return conn, err
}

conn.Cluster = cluster
Expand All @@ -54,36 +54,42 @@ func getDbConnection(cred Credentials) (conn CbConnection) {

err = conn.Bucket.WaitUntilReady(5*time.Second, nil)
if err != nil {
log.Fatal(err)
return
return conn, err
}

conn.Scope = conn.Bucket.Scope(cred.Cb_scope)
return conn
return conn, nil
}

func queryWithSQLFile(scope *gocb.Scope, file string) (jsonOut []string) {
func queryWithSQLFile(scope *gocb.Scope, file string) (jsonOut []string, err error) {
fileContent, err := os.ReadFile(file)
if err != nil {
log.Fatal(err)
log.Println("Error reading file:", err)
return nil, err
}

// Convert []byte to string
text := string(fileContent)
fmt.Println(text)

return queryWithSQLStringSA(scope, text)
result, err := queryWithSQLStringSA(scope, text)
if err != nil {
return nil, err
}
return result, nil
}

func queryWithSQLStringSA(scope *gocb.Scope, text string) (rv []string) {
func queryWithSQLStringSA(scope *gocb.Scope, text string) (rv []string, err error) {
log.Println("queryWithSQLStringSA(\n" + text + "\n)")

queryResult, err := scope.Query(
fmt.Sprintf(text),
&gocb.QueryOptions{Adhoc: true},
&gocb.QueryOptions{
Adhoc: true,
Timeout: 300 * time.Second, // Set an explicit timeout
},
)
if err != nil {
log.Fatal(err)
return nil, err
}

// Interfaces for handling streaming return values
Expand All @@ -95,23 +101,23 @@ func queryWithSQLStringSA(scope *gocb.Scope, text string) (rv []string) {
var row interface{}
err := queryResult.Row(&row)
if err != nil {
log.Fatal(err)
return nil, err
}
retValues = append(retValues, row.(string))
}

return retValues
return retValues, nil
}

func queryWithSQLStringFA(scope *gocb.Scope, text string) (rv []float64) {
func queryWithSQLStringFA(scope *gocb.Scope, text string) (rv []float64, err error) {
log.Println("queryWithSQLStringFA(\n" + text + "\n)")

queryResult, err := scope.Query(
fmt.Sprintf(text),
&gocb.QueryOptions{Adhoc: true},
&gocb.QueryOptions{Adhoc: true, Timeout: 300 * time.Second},
)
if err != nil {
log.Fatal(err)
return nil, err
}

retValues := make([]float64, 0)
Expand All @@ -122,23 +128,23 @@ func queryWithSQLStringFA(scope *gocb.Scope, text string) (rv []float64) {
var row interface{}
err := queryResult.Row(&row)
if err != nil {
log.Fatal(err)
return nil, err
}
retValues = append(retValues, row.(float64))
}

return retValues
return retValues, nil
}

func queryWithSQLStringIA(scope *gocb.Scope, text string) (rv []int) {
func queryWithSQLStringIA(scope *gocb.Scope, text string) (rv []int, err error) {
log.Println("queryWithSQLStringFA(\n" + text + "\n)")

queryResult, err := scope.Query(
fmt.Sprintf(text),
&gocb.QueryOptions{Adhoc: true},
&gocb.QueryOptions{Adhoc: true, Timeout: 300 * time.Second},
)
if err != nil {
log.Fatal(err)
return nil, err
}

retValues := make([]int, 0)
Expand All @@ -149,28 +155,28 @@ func queryWithSQLStringIA(scope *gocb.Scope, text string) (rv []int) {
var row interface{}
err := queryResult.Row(&row)
if err != nil {
log.Fatal(err)
return nil, err
}
switch row.(type) {
switch v := row.(type) {
case float64:
retValues = append(retValues, int(row.(float64)))
retValues = append(retValues, int(v))
case int:
retValues = append(retValues, row.(int))
retValues = append(retValues, v)
}
}

return retValues
return retValues, nil
}

func queryWithSQLStringMAP(scope *gocb.Scope, text string) (jsonOut []interface{}) {
func queryWithSQLStringMAP(scope *gocb.Scope, text string) (jsonOut []interface{}, err error) {
log.Println("queryWithSQLStringMAP(\n" + text + "\n)")

queryResult, err := scope.Query(
fmt.Sprintf(text),
&gocb.QueryOptions{Adhoc: true},
&gocb.QueryOptions{Adhoc: true, Timeout: 300 * time.Second},
)
if err != nil {
log.Fatal(err)
return nil, err
}

rows := make([]interface{}, 0)
Expand All @@ -179,18 +185,18 @@ func queryWithSQLStringMAP(scope *gocb.Scope, text string) (jsonOut []interface{
var row interface{}
err := queryResult.Row(&row)
if err != nil {
log.Fatal(err)
return nil, err
}
m := row.(map[string]interface{})
rows = append(rows, m)
}
return rows
return rows, nil
}

func queryWithSQLFileJustPrint(scope *gocb.Scope, file string) {
func queryWithSQLFileJustPrint(scope *gocb.Scope, file string) error {
fileContent, err := os.ReadFile(file)
if err != nil {
log.Fatal(err)
return err
}

// Convert []byte to string
Expand All @@ -199,22 +205,22 @@ func queryWithSQLFileJustPrint(scope *gocb.Scope, file string) {

queryResult, err := scope.Query(
fmt.Sprintf(text),
&gocb.QueryOptions{Adhoc: true},
&gocb.QueryOptions{Adhoc: true, Timeout: 300 * time.Second},
)
if err != nil {
log.Fatal(err)
} else {
printQueryResult(queryResult)
return err
}
return printQueryResult(queryResult)
}

func printQueryResult(queryResult *gocb.QueryResult) {
func printQueryResult(queryResult *gocb.QueryResult) error {
for queryResult.Next() {
var result interface{}
err := queryResult.Row(&result)
if err != nil {
log.Fatal(err)
return err
}
fmt.Println(result)
}
return nil
}
66 changes: 52 additions & 14 deletions meta_update_middleware/meta-update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"os"
"time"
Expand Down Expand Up @@ -63,8 +62,6 @@ func main() {
log.Println("meta-update, settings file:" + settingsFilePath + ",credentials file:" + credentialsFilePath + ",app:[all apps in settings file]")
}

conf := ConfigJSON{}

conf, err := parseConfig(settingsFilePath)
if err != nil {
log.Fatal("Unable to parse config")
Expand All @@ -73,7 +70,10 @@ func main() {

credentials := getCredentials(credentialsFilePath)

conn := getDbConnection(credentials)
conn, err := getDbConnection(credentials)
if err != nil {
log.Fatalf("Failed to connect to database: %v", err)
}

//testGetSingleCTC(conn)
//testGetCTCCount(connSrc)
Expand All @@ -89,20 +89,28 @@ func main() {
updateMedataForAppDocType(conn, conf.Metadata[ds].Name, conf.Metadata[ds].App, conf.Metadata[ds].DocType[dt], conf.Metadata[ds].SubDocType)
}
}
log.Println(fmt.Sprintf("\tmeta update finished in %v", time.Since(start)))
log.Printf("\tmeta update finished in %v", time.Since(start))
}

func updateMedataForAppDocType(conn CbConnection, name string, app string, doctype string, subDocType string) {
log.Println("updateMedataForAppDocType(" + name + "," + doctype + ")")

// get needed models
models := getModels(conn, name, app, doctype, subDocType)
models, err := getModels(conn, name, app, doctype, subDocType)
if err != nil {
log.Printf("Error getting models: %v", err)
return
}
log.Println("models:")
printStringArray(models)

// get models having metadata but no data (remove metadata for these)
// (note 'like %' is changed to 'like %25')
models_with_metatada_but_no_data := getModelsNoData(conn, name, app, doctype, subDocType)
models_with_metatada_but_no_data, err := getModelsNoData(conn, name, app, doctype, subDocType)
if err != nil {
log.Printf("Error getting models with metadata but no data: %v", err)
return
}
log.Println("models_with_metatada_but_no_data:")
printStringArray(models_with_metatada_but_no_data)

Expand Down Expand Up @@ -132,20 +140,50 @@ func updateMedataForAppDocType(conn CbConnection, name string, app string, docty
metadata.Updated = 0

for i, m := range models {
var err error
var thresholds []string
model := Model{Name: m}
thresholds := getDistinctThresholds(conn, name, app, doctype, subDocType, m)
thresholds, err = getDistinctThresholds(conn, name, app, doctype, subDocType, m)
if err != nil {
log.Printf("Error getting distinct thresholds: %v", err)
continue
}
log.Println(thresholds)
fcstLen := getDistinctFcstLen(conn, name, app, doctype, subDocType, m)
fcstLen, err := getDistinctFcstLen(conn, name, app, doctype, subDocType, m)
if err != nil {
log.Printf("Error getting distinct forecast lengths: %v", err)
continue
}
log.Println(fcstLen)
region := getDistinctRegion(conn, name, app, doctype, subDocType, m)
region, err := getDistinctRegion(conn, name, app, doctype, subDocType, m)
if err != nil {
log.Printf("Error getting distinct region: %v", err)
continue
}
log.Println(region)
displayText := getDistinctDisplayText(conn, name, app, doctype, subDocType, m)
displayText, err := getDistinctDisplayText(conn, name, app, doctype, subDocType, m)
if err != nil {
log.Printf("Error getting distinct display text: %v", err)
continue
}
log.Println(displayText)
displayCategory := getDistinctDisplayCategory(conn, name, app, doctype, subDocType, m)
displayCategory, err := getDistinctDisplayCategory(conn, name, app, doctype, subDocType, m)
if err != nil {
log.Printf("Error getting distinct display category: %v", err)
continue
}
log.Println(displayCategory)
displayOrder := getDistinctDisplayOrder(conn, name, app, doctype, subDocType, m, i)
displayOrder, err := getDistinctDisplayOrder(conn, name, app, doctype, subDocType, m, i)
if err != nil {
log.Printf("Error getting distinct display order: %v", err)
continue
}
log.Println(displayOrder)
minMaxCountFloor := getMinMaxCountFloor(conn, name, app, doctype, subDocType, m)
minMaxCountFloor, err := getMinMaxCountFloor(conn, name, app, doctype, subDocType, m)
if err != nil {
log.Printf("Error getting min/max/count/floor: %v", err)
continue
}
log.Println(jsonPrettyPrintStruct(minMaxCountFloor[0].(map[string]interface{})))

// ./sqls/getDistinctThresholds.sql returns list of variables for SUMS DocType, like in Surface
Expand Down
Loading
Loading