From dd29d588b5f86056a8c8bc6263eeca09298500cb Mon Sep 17 00:00:00 2001 From: Daniel Gruber Date: Mon, 28 Oct 2024 11:32:46 +0100 Subject: [PATCH] EH: qstat -g t parsing. Test example. --- examples/testexample/.gitignore | 1 + examples/testexample/go.mod | 13 ++ examples/testexample/go.sum | 38 +++++ examples/testexample/testexample.go | 256 ++++++++++++++++++++++++++++ pkg/qstat/v9.0/parser.go | 149 +++++++++++++++- pkg/qstat/v9.0/qstat.go | 4 +- pkg/qstat/v9.0/qstat_impl.go | 4 +- pkg/qstat/v9.0/qstat_impl_test.go | 4 +- pkg/qstat/v9.0/types.go | 47 ++--- 9 files changed, 488 insertions(+), 28 deletions(-) create mode 100644 examples/testexample/.gitignore create mode 100644 examples/testexample/go.mod create mode 100644 examples/testexample/go.sum create mode 100644 examples/testexample/testexample.go diff --git a/examples/testexample/.gitignore b/examples/testexample/.gitignore new file mode 100644 index 0000000..5928b89 --- /dev/null +++ b/examples/testexample/.gitignore @@ -0,0 +1 @@ +testexample diff --git a/examples/testexample/go.mod b/examples/testexample/go.mod new file mode 100644 index 0000000..cfd68d1 --- /dev/null +++ b/examples/testexample/go.mod @@ -0,0 +1,13 @@ +module github.com/hpc-gridware/go-clusterscheduler/examples/testexample + +go 1.23.1 + +replace github.com/hpc-gridware/go-clusterscheduler => ../.. + +require ( + github.com/hpc-gridware/go-clusterscheduler v0.0.0-20241027163340-55dac298d370 + go.uber.org/zap v1.27.0 + google.golang.org/protobuf v1.35.1 +) + +require go.uber.org/multierr v1.10.0 // indirect diff --git a/examples/testexample/go.sum b/examples/testexample/go.sum new file mode 100644 index 0000000..64ed0e3 --- /dev/null +++ b/examples/testexample/go.sum @@ -0,0 +1,38 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0= +github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/testexample/testexample.go b/examples/testexample/testexample.go new file mode 100644 index 0000000..6dd8fcf --- /dev/null +++ b/examples/testexample/testexample.go @@ -0,0 +1,256 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + qacct "github.com/hpc-gridware/go-clusterscheduler/pkg/qacct/v9.0" + qstat "github.com/hpc-gridware/go-clusterscheduler/pkg/qstat/v9.0" + "google.golang.org/protobuf/types/known/timestamppb" + + "go.uber.org/zap" +) + +var qacctClient qacct.QAcct +var qstatClient qstat.QStat + +var log *zap.Logger + +func init() { + var err error + log, _ = zap.NewProduction() + qacctClient, err = qacct.NewCommandLineQAcct(qacct.CommandLineQAcctConfig{}) + if err != nil { + log.Fatal("Failed to initialize qacct client", zap.String("error", + err.Error())) + } + qstatClient, err = qstat.NewCommandLineQstat(qstat.CommandLineQStatConfig{}) + if err != nil { + log.Fatal("Failed to initialize qstat client", zap.String("error", + err.Error())) + } +} + +func main() { + run(context.Background()) +} + +func run(ctx context.Context) { + defer log.Sync() + alreadySent := map[string]struct{}{} + + for { + select { + case <-ctx.Done(): + log.Info("Context cancelled, stopping ClusterScheduler") + return + default: + finishedJobs, err := GetFinishedJobs() + if err != nil { + log.Error("Error getting finished jobs", zap.String("error", + err.Error())) + } + + runningJobs, err := GetRunningJobs() + if err != nil { + log.Error("Error getting running jobs", zap.String("error", + err.Error())) + } + + allJobs := append(finishedJobs, runningJobs...) + + var protoJobs []*SimpleJob + for _, job := range allJobs { + if _, ok := alreadySent[job.JobId]; ok { + continue + } + protoJobs = append(protoJobs, job) + } + + _, err = SendJobs(ctx, protoJobs) + if err != nil { + log.Warn("Error ingesting jobs", zap.String("error", + err.Error())) + } + + for _, job := range finishedJobs { + alreadySent[job.JobId] = struct{}{} + } + + select { + case <-ctx.Done(): + log.Info("Context cancelled, stopping") + return + case <-time.After(10 * time.Second): + } + } + } +} + +type SimpleJob struct { + JobId string `json:"job_id"` + // Cluster represents the queue name + Cluster string `json:"cluster"` + JobName string `json:"job_name"` + // Partition represents the parallel environment + Partition string `json:"partition"` + Account string `json:"account"` + User string `json:"user"` + State string `json:"state"` + ExitCode string `json:"exit_code"` + Submit *timestamppb.Timestamp `json:"submit"` + Start *timestamppb.Timestamp `json:"start"` + End *timestamppb.Timestamp `json:"end"` + MasterNode string `json:"master_node"` +} + +func GetFinishedJobs() ([]*SimpleJob, error) { + // Use qacct NativeSpecification to get finished jobs + qacctOutput, err := qacctClient.NativeSpecification([]string{"-j", "*"}) + if err != nil { + return nil, fmt.Errorf("error running qacct command: %v", err) + } + + jobs, err := qacct.ParseQAcctJobOutput(qacctOutput) + if err != nil { + return nil, fmt.Errorf("error parsing qacct output: %v", err) + } + // convert to SimpleJob format + simpleJobs := make([]*SimpleJob, len(jobs)) + for i, job := range jobs { + state := fmt.Sprintf("%d", job.ExitStatus) + if state == "0" { + state = "done" + } else { + state = "failed" + } + simpleJobs[i] = &SimpleJob{ + // ignore job arrays for now + JobId: fmt.Sprintf("%d", job.JobNumber), + Cluster: job.QName, + JobName: job.JobName, + Partition: job.GrantedPE, + Account: job.Account, + User: job.Owner, + State: state, + ExitCode: fmt.Sprintf("%d", job.ExitStatus), + Submit: parseTimestamp(job.QSubTime), + Start: parseTimestamp(job.StartTime), + End: parseTimestamp(job.EndTime), + MasterNode: job.HostName, + } + } + return simpleJobs, nil +} + +func GetRunningJobs() ([]*SimpleJob, error) { + + qstatOverview, err := qstatClient.NativeSpecification([]string{"-g", "t"}) + if err != nil { + return nil, fmt.Errorf("error running qstat command: %v", err) + } + jobsByTask, err := qstat.ParseGroupByTask(qstatOverview) + if err != nil { + return nil, fmt.Errorf("error parsing qstat output: %v", err) + } + + type State struct { + QueueName string + State string + MasterNode string + } + + stateMap := map[int]State{} + + for _, job := range jobsByTask { + // we are only interested in serial and parallel jobs (no arrays) + jq := strings.Split(job.Queue, "@") + if len(jq) == 2 { + js, exists := stateMap[job.JobID] + if !exists { + js = State{ + QueueName: jq[0], + State: job.State, + MasterNode: jq[1], + } + } + if job.Master == "MASTER" { + // this is the master task of a parallel job + js.MasterNode = jq[1] + stateMap[job.JobID] = js + } + continue + } + stateMap[job.JobID] = State{ + QueueName: job.Queue, + State: job.State, + } + } + + // get running jobs + qstatOutput, err := qstatClient.NativeSpecification([]string{"-j", "*"}) + if err != nil { + return nil, fmt.Errorf("error running qstat command: %v", err) + } + + jobs, err := qstat.ParseSchedulerJobInfo(qstatOutput) + if err != nil { + return nil, fmt.Errorf("error parsing qstat output: %v", err) + } + + // convert to SimpleJob format + simpleJobs := make([]*SimpleJob, len(jobs)) + for i, job := range jobs { + state := stateMap[job.JobNumber].State + if state == "" { + state = "running" + } + simpleJobs[i] = &SimpleJob{ + JobId: fmt.Sprintf("%d", job.JobNumber), + Cluster: stateMap[job.JobNumber].QueueName, + JobName: job.JobName, + Partition: strings.Split(job.ParallelEnvironment, " ")[0], // PE + Account: job.Account, + User: job.Owner, + State: state, + ExitCode: "", + MasterNode: stateMap[job.JobNumber].MasterNode, + } + if strings.Contains(stateMap[job.JobNumber].State, "q") { + simpleJobs[i].Submit = parseTimestamp(job.SubmissionTime) + } else { + simpleJobs[i].Start = parseTimestamp(job.SubmissionTime) + } + } + return simpleJobs, nil +} + +func SendJobs(ctx context.Context, jobs []*SimpleJob) (int, error) { + log.Info("Sending jobs", zap.Int("jobs", len(jobs))) + // Print the jobs + for _, job := range jobs { + // pretty print JSON + json, err := json.MarshalIndent(job, "", " ") + if err != nil { + return 0, fmt.Errorf("error marshalling job: %v", err) + } + fmt.Println(string(json)) + } + return len(jobs), nil +} + +// 2024-10-24 09:49:59.911136 +func parseTimestamp(s string) *timestamppb.Timestamp { + loc, err := time.LoadLocation("Local") + if err != nil { + return nil + } + t, err := time.ParseInLocation("2006-01-02 15:04:05.999999", s, loc) + if err != nil { + return nil + } + return timestamppb.New(t) +} diff --git a/pkg/qstat/v9.0/parser.go b/pkg/qstat/v9.0/parser.go index 562829d..4ec3207 100644 --- a/pkg/qstat/v9.0/parser.go +++ b/pkg/qstat/v9.0/parser.go @@ -20,12 +20,151 @@ package qstat import ( + "bufio" + "fmt" + "log" "strconv" "strings" ) -// parseJobs parses the input text into a slice of SchedulerJobInfo instances. -func parseJobs(input string) ([]SchedulerJobInfo, error) { +// ParseGroupByTask parses the input text into a slice of +// SchedulerJobInfo instances (qstat -g t output). +func ParseGroupByTask(input string) ([]ParallelJobTask, error) { + + // These are examples of the output format which needs to be parsed: + + /* job-ID prior name user state submit/start at queue master ja-task-ID + ------------------------------------------------------------------------------------------------------------------ + 14 0.50500 sleep root r 2024-10-28 07:21:41 all.q@master MASTER + 17 0.60500 sleep root r 2024-10-28 07:29:24 all.q@master MASTER + all.q@master SLAVE + all.q@master SLAVE + all.q@master SLAVE + 12 0.50500 sleep root qw 2024-10-28 07:17:34 + */ + + /* + job-ID prior name user state submit/start at queue master ja-task-ID + ------------------------------------------------------------------------------------------------------------------ + 14 0.50500 sleep root r 2024-10-28 07:21:41 all.q@master MASTER + 15 0.50500 sleep root r 2024-10-28 07:26:14 all.q@master MASTER 1 + 15 0.50500 sleep root r 2024-10-28 07:26:14 all.q@master MASTER 3 + 15 0.50500 sleep root r 2024-10-28 07:26:14 all.q@master MASTER 5 + 17 0.60500 sleep root qw 2024-10-28 07:27:50 + 12 0.50500 sleep root qw 2024-10-28 07:17:34 + 15 0.50500 sleep root qw 2024-10-28 07:26:14 7-99:2 + */ + return parseFixedWidthJobs(input) + +} +func parseFixedWidthJobs(input string) ([]ParallelJobTask, error) { + var tasks []ParallelJobTask + + // Correct column positions based on your description + columnPositions := []struct { + start int + end int + }{ + {start: 0, end: 8}, // job-ID + {start: 8, end: 16}, // prior + {start: 16, end: 26}, // name + {start: 26, end: 38}, // user + {start: 38, end: 44}, // state + {start: 44, end: 65}, // submit/start at + {start: 65, end: 94}, // queue + {start: 94, end: 104}, // master + {start: 104, end: 112}, // ja-task-ID (if exists) + } + + scanner := bufio.NewScanner(strings.NewReader(input)) + if !scanner.Scan() || !scanner.Scan() { + return nil, fmt.Errorf("input doesn't contain header or dashed line") + } + + var currentJob *ParallelJobTask + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + continue + } + + fields := make([]string, len(columnPositions)) + for i, pos := range columnPositions { + if pos.start < len(line) { + end := pos.end + if end > len(line) { + end = len(line) + } + fields[i] = strings.TrimSpace(line[pos.start:end]) + } else { + fields[i] = "" + } + } + + if isContinuationLine(fields) { + if currentJob != nil && len(fields) > 6 { + currentJob.Queue = fields[6] + currentJob.Master = fields[7] + currentJob.Slots++ + } + } else { + jobInfo, err := parseFixedWidthJobInfo(fields) + if err != nil { + log.Println("Skipping line due to parsing error:", err) + continue + } + + task := ParallelJobTask{JobInfo: *jobInfo} + if fields[7] != "" { + task.Master = fields[7] + } + if len(fields) > 8 && fields[8] != "" { + task.JobInfo.TaskID = fields[8] + } + tasks = append(tasks, task) + currentJob = &tasks[len(tasks)-1] + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("scanner error: %w", err) + } + + return tasks, nil +} + +func isContinuationLine(fields []string) bool { + return len(fields[0]) == 0 +} + +func parseFixedWidthJobInfo(fields []string) (*JobInfo, error) { + jobID, err := strconv.Atoi(fields[0]) + if err != nil { + return nil, fmt.Errorf("invalid job ID: %v", err) + } + + priority, err := strconv.ParseFloat(fields[1], 64) + if err != nil { + return nil, fmt.Errorf("invalid priority: %v", err) + } + + jobInfo := &JobInfo{ + JobID: jobID, + Priority: priority, + Name: fields[2], + User: fields[3], + State: fields[4], + SubmitStartAt: fields[5], + Queue: fields[6], + Slots: 1, + } + + return jobInfo, nil +} + +// ParseSchedulerJobInfo parses the input text into a slice of +// SchedulerJobInfo instances (qstat -j output). +func ParseSchedulerJobInfo(input string) ([]SchedulerJobInfo, error) { var jobs []SchedulerJobInfo blocks := strings.Split(input, "\n==============================================================\n") @@ -66,6 +205,10 @@ func parseJob(block string) (SchedulerJobInfo, error) { info.ExecFile = value case "submission_time": info.SubmissionTime = value + case "submit_cmd_line": + info.SubmitCmdLine = value + case "effective_submit_cmd_line": + info.EffectiveSubmitCmdLine = value case "owner": info.Owner = value case "uid": @@ -98,6 +241,8 @@ func parseJob(block string) (SchedulerJobInfo, error) { info.JobArgs = value case "script_file": info.ScriptFile = value + case "parallel_environment": + info.ParallelEnvironment = value case "binding": info.Binding = value case "scheduling info": diff --git a/pkg/qstat/v9.0/qstat.go b/pkg/qstat/v9.0/qstat.go index d442d23..045c1b7 100644 --- a/pkg/qstat/v9.0/qstat.go +++ b/pkg/qstat/v9.0/qstat.go @@ -19,6 +19,8 @@ package qstat +import "context" + // QStat defines the methods for interacting with the Open Cluster Scheduler // to retrieve job and queue status information using the qstat command. // @@ -29,7 +31,7 @@ type QStat interface { // job with the given job ID. It waits up to 3 seconds for the job status // to appear in the system. When the job left the system (due to job end), // the channel will be closed. - WatchJob(jobId int) (chan SchedulerJobInfo, error) + WatchJobs(ctx context.Context, jobIds []int64) (chan SchedulerJobInfo, error) // NativeSpecification calls qstat with the native specification of args // and returns the raw output NativeSpecification(args []string) (string, error) diff --git a/pkg/qstat/v9.0/qstat_impl.go b/pkg/qstat/v9.0/qstat_impl.go index 383b16e..230e6c3 100644 --- a/pkg/qstat/v9.0/qstat_impl.go +++ b/pkg/qstat/v9.0/qstat_impl.go @@ -77,7 +77,7 @@ func (q *QStatImpl) WatchJobs(ctx context.Context, jobIds []int64) (chan Schedul continue } // found job - jobs, err = parseJobs(out) + jobs, err = ParseSchedulerJobInfo(out) if err != nil { return nil, fmt.Errorf("error parsing jobs: %w", err) } @@ -108,7 +108,7 @@ func (q *QStatImpl) WatchJobs(ctx context.Context, jobIds []int64) (chan Schedul break } // found jobs - jobs, err = parseJobs(out) + jobs, err = ParseSchedulerJobInfo(out) if err != nil || len(jobs) == 0 { // all jobs left the system break diff --git a/pkg/qstat/v9.0/qstat_impl_test.go b/pkg/qstat/v9.0/qstat_impl_test.go index b51e092..625443d 100644 --- a/pkg/qstat/v9.0/qstat_impl_test.go +++ b/pkg/qstat/v9.0/qstat_impl_test.go @@ -31,7 +31,9 @@ var _ = Describe("QstatImpl", func() { Context("NativeSpecification", func() { It("should return the command line", func() { - q, err := qstat.NewCommandLineQstat( + var err error + var q qstat.QStat + q, err = qstat.NewCommandLineQstat( qstat.CommandLineQStatConfig{ DryRun: true, }) diff --git a/pkg/qstat/v9.0/types.go b/pkg/qstat/v9.0/types.go index 37c17af..dd4c511 100644 --- a/pkg/qstat/v9.0/types.go +++ b/pkg/qstat/v9.0/types.go @@ -150,28 +150,31 @@ type JobPriority struct { // SchedulerJobInfo represents detailed information about a scheduled job // retrieved with the qstat -j command. type SchedulerJobInfo struct { - JobNumber int `json:"job_number"` - ExecFile string `json:"exec_file"` - SubmissionTime string `json:"submission_time"` - Owner string `json:"owner"` - UID int `json:"uid"` - Group string `json:"group"` - GID int `json:"gid"` - SgeOHome string `json:"sge_o_home"` - SgeOPath string `json:"sge_o_path"` - SgeOWorkDir string `json:"sge_o_workdir"` - SgeOHost string `json:"sge_o_host"` - Account string `json:"account"` - MailList string `json:"mail_list"` - Notify bool `json:"notify"` - JobName string `json:"job_name"` - JobShare int `json:"jobshare"` - EnvList string `json:"env_list"` - JobArgs string `json:"job_args"` - ScriptFile string `json:"script_file"` - Binding string `json:"binding"` - Usage []UsageDetail `json:"usage"` - SchedulingInfo string `json:"scheduling_info"` + JobNumber int `json:"job_number"` + ExecFile string `json:"exec_file"` + SubmissionTime string `json:"submission_time"` + SubmitCmdLine string `json:"submit_cmd_line"` + EffectiveSubmitCmdLine string `json:"effective_submit_cmd_line"` + Owner string `json:"owner"` + UID int `json:"uid"` + Group string `json:"group"` + GID int `json:"gid"` + SgeOHome string `json:"sge_o_home"` + SgeOPath string `json:"sge_o_path"` + SgeOWorkDir string `json:"sge_o_workdir"` + SgeOHost string `json:"sge_o_host"` + Account string `json:"account"` + MailList string `json:"mail_list"` + Notify bool `json:"notify"` + JobName string `json:"job_name"` + JobShare int `json:"jobshare"` + EnvList string `json:"env_list"` + JobArgs string `json:"job_args"` + ScriptFile string `json:"script_file"` + ParallelEnvironment string `json:"parallel_environment"` + Binding string `json:"binding"` + Usage []UsageDetail `json:"usage"` + SchedulingInfo string `json:"scheduling_info"` } type UsageDetail struct {