Skip to content

Commit 06378f1

Browse files
dgruberdgruber
authored
EH: Version specific subdirectories (#19)
* EH: Update to OCS 9.0.1 container; qsub / qacct / qstat with native spec * EH: Move APIs to version specific subdirs. Simple call wrappers with native specification
1 parent bc9350d commit 06378f1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+2061
-12
lines changed

‎cmd/adapter/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import (
2525

2626
"github.com/gorilla/mux"
2727
"github.com/hpc-gridware/go-clusterscheduler/pkg/adapter"
28-
"github.com/hpc-gridware/go-clusterscheduler/pkg/qconf"
28+
qconf "github.com/hpc-gridware/go-clusterscheduler/pkg/qconf/v9.0"
2929
)
3030

3131
func main() {

‎cmd/diag/diag.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"encoding/json"
55
"fmt"
66

7-
"github.com/hpc-gridware/go-clusterscheduler/pkg/qconf"
7+
qconf "github.com/hpc-gridware/go-clusterscheduler/pkg/qconf/v9.0"
88
)
99

1010
func main() {

‎cmd/simulator/dump.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import (
2525
"os"
2626
"time"
2727

28-
"github.com/hpc-gridware/go-clusterscheduler/pkg/qconf"
28+
qconf "github.com/hpc-gridware/go-clusterscheduler/pkg/qconf/v9.0"
2929
"github.com/spf13/cobra"
3030
)
3131

‎cmd/simulator/run.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
"path/filepath"
2929
"time"
3030

31-
"github.com/hpc-gridware/go-clusterscheduler/pkg/qconf"
31+
qconf "github.com/hpc-gridware/go-clusterscheduler/pkg/qconf/v9.0"
3232
"github.com/spf13/cobra"
3333
)
3434

‎examples/simplejoblifecycle/simplejoblifecycle.go

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*___INFO__MARK_BEGIN__*/
2+
/*************************************************************************
3+
* Copyright 2024 HPC-Gridware GmbH
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
************************************************************************/
18+
/*___INFO__MARK_END__*/
19+
20+
package main
21+
22+
import (
23+
"context"
24+
"encoding/json"
25+
"fmt"
26+
"os"
27+
"time"
28+
29+
qacct "github.com/hpc-gridware/go-clusterscheduler/pkg/qacct/v9.0"
30+
qstat "github.com/hpc-gridware/go-clusterscheduler/pkg/qstat/v9.0"
31+
qsub "github.com/hpc-gridware/go-clusterscheduler/pkg/qsub/v9.0"
32+
)
33+
34+
func main() {
35+
36+
ctx := context.Background()
37+
38+
qs, err := qsub.NewCommandLineQSub(qsub.CommandLineQSubConfig{})
39+
if err != nil {
40+
fmt.Printf("error creating qsub client: %v\n", err)
41+
os.Exit(1)
42+
}
43+
44+
// watching the job with qstat
45+
qstat, err := qstat.NewCommandLineQstat(qstat.CommandLineQStatConfig{})
46+
if err != nil {
47+
fmt.Printf("error creating qstat client: %v\n", err)
48+
os.Exit(1)
49+
}
50+
51+
qacct, err := qacct.NewCommandLineQAcct(qacct.CommandLineQAcctConfig{})
52+
if err != nil {
53+
fmt.Printf("error creating qacct client: %v\n", err)
54+
os.Exit(1)
55+
}
56+
57+
jobId, _, err := qs.Submit(ctx, qsub.JobOptions{
58+
Command: "sleep",
59+
CommandArgs: []string{"10"},
60+
Binary: qsub.ToPtr(true),
61+
})
62+
if err != nil {
63+
fmt.Printf("error submitting job: %v\n", err)
64+
os.Exit(1)
65+
}
66+
fmt.Printf("submitted job with id %d\n", jobId)
67+
68+
jobInfoCh, err := qstat.WatchJobs(ctx, []int64{jobId})
69+
if err != nil {
70+
fmt.Printf("error watching job: %v\n", err)
71+
os.Exit(1)
72+
}
73+
74+
// watch with qstat until the job is done
75+
for jobInfo := range jobInfoCh {
76+
// nicely formatted JSON output
77+
json, err := json.MarshalIndent(jobInfo, "", " ")
78+
if err != nil {
79+
fmt.Printf("error marshalling job info: %v\n", err)
80+
} else {
81+
fmt.Printf("job info: %s\n", string(json))
82+
}
83+
}
84+
85+
// when the job is done, it is out-of-scope of the scheduler
86+
fmt.Printf("job left the system\n")
87+
88+
// it may take a while until the accounting information is available
89+
<-time.After(2 * time.Second)
90+
91+
// get accounting information with qacct
92+
jobAcct, err := qacct.ShowJobDetails([]int64{jobId})
93+
if err != nil {
94+
fmt.Printf("error getting accounting information: %v\n", err)
95+
os.Exit(1)
96+
}
97+
98+
// nicely formatted JSON output
99+
json, err := json.MarshalIndent(jobAcct, "", " ")
100+
if err != nil {
101+
fmt.Printf("error marshalling accounting information: %v\n", err)
102+
} else {
103+
fmt.Printf("job accounting information: %s\n", string(json))
104+
}
105+
}

‎pkg/cs/clusterscheduler.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ package cs
2222
import (
2323
"fmt"
2424

25-
"github.com/hpc-gridware/go-clusterscheduler/pkg/qconf"
25+
qconf "github.com/hpc-gridware/go-clusterscheduler/pkg/qconf/v9.0"
2626
)
2727

2828
// ClusterScheduler defines the methods that a cluster scheduler connection should implement.

‎pkg/cs/command_conn.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ package cs
2222
import (
2323
"path/filepath"
2424

25-
"github.com/hpc-gridware/go-clusterscheduler/pkg/qconf"
25+
qconf "github.com/hpc-gridware/go-clusterscheduler/pkg/qconf/v9.0"
2626
)
2727

2828
type CommandLineInterface struct {

‎pkg/qacct/memory.go renamed to ‎pkg/qacct/v9.0/memory.go

File renamed without changes.

‎pkg/qacct/v9.0/parse.go

+177
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
/*___INFO__MARK_BEGIN__*/
2+
/*************************************************************************
3+
* Copyright 2024 HPC-Gridware GmbH
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
************************************************************************/
18+
/*___INFO__MARK_END__*/
19+
20+
package qacct
21+
22+
import (
23+
"bufio"
24+
"strconv"
25+
"strings"
26+
)
27+
28+
func ParseQacctJobOutputWithScanner(scanner *bufio.Scanner) ([]JobDetail, error) {
29+
var jobs []JobDetail
30+
var job JobDetail
31+
32+
for scanner.Scan() {
33+
line := scanner.Text()
34+
if strings.HasPrefix(line, "==============================================================") {
35+
if job.JobNumber != 0 {
36+
jobs = append(jobs, job)
37+
}
38+
job = JobDetail{}
39+
continue
40+
}
41+
42+
parts := strings.SplitN(line, " ", 2)
43+
if len(parts) < 2 {
44+
continue
45+
}
46+
47+
key := strings.TrimSpace(parts[0])
48+
value := strings.TrimSpace(parts[1])
49+
50+
switch key {
51+
case "qname":
52+
job.QName = value
53+
case "hostname":
54+
job.HostName = value
55+
case "group":
56+
job.Group = value
57+
case "owner":
58+
job.Owner = value
59+
case "project":
60+
job.Project = value
61+
case "department":
62+
job.Department = value
63+
case "jobname":
64+
job.JobName = value
65+
case "jobnumber":
66+
job.JobNumber = parseInt64(value)
67+
case "taskid":
68+
job.TaskID = parseInt64(value)
69+
case "pe_taskid":
70+
job.PETaskID = value
71+
case "account":
72+
job.Account = value
73+
case "priority":
74+
job.Priority = parseInt64(value)
75+
case "qsub_time":
76+
job.QSubTime = value
77+
case "submit_cmd_line":
78+
job.SubmitCommandLine = value
79+
case "start_time":
80+
job.StartTime = value
81+
case "end_time":
82+
job.EndTime = value
83+
case "granted_pe":
84+
job.GrantedPE = value
85+
case "slots":
86+
job.Slots = parseInt64(value)
87+
case "failed":
88+
job.Failed = parseInt64(value)
89+
case "exit_status":
90+
job.ExitStatus = parseInt64(value)
91+
case "ru_wallclock":
92+
job.RuWallClock = parseFloat(value)
93+
case "ru_utime":
94+
job.RuUTime = parseFloat(value)
95+
case "ru_stime":
96+
job.RuSTime = parseFloat(value)
97+
case "ru_maxrss":
98+
job.RuMaxRSS = parseInt64(value)
99+
case "ru_ixrss":
100+
job.RuIXRSS = parseInt64(value)
101+
case "ru_ismrss":
102+
job.RuISMRSS = parseInt64(value)
103+
case "ru_idrss":
104+
job.RuIDRSS = parseInt64(value)
105+
case "ru_isrss":
106+
job.RuISRss = parseInt64(value)
107+
case "ru_minflt":
108+
job.RuMinFlt = parseInt64(value)
109+
case "ru_majflt":
110+
job.RuMajFlt = parseInt64(value)
111+
case "ru_nswap":
112+
job.RuNSwap = parseInt64(value)
113+
case "ru_inblock":
114+
job.RuInBlock = parseInt64(value)
115+
case "ru_oublock":
116+
job.RuOuBlock = parseInt64(value)
117+
case "ru_msgsnd":
118+
job.RuMsgSend = parseInt64(value)
119+
case "ru_msgrcv":
120+
job.RuMsgRcv = parseInt64(value)
121+
case "ru_nsignals":
122+
job.RuNSignals = parseInt64(value)
123+
case "ru_nvcsw":
124+
job.RuNVCSw = parseInt64(value)
125+
case "ru_nivcsw":
126+
job.RuNiVCSw = parseInt64(value)
127+
case "wallclock":
128+
job.WallClock = parseFloat(value)
129+
case "cpu":
130+
job.CPU = parseFloat(value)
131+
case "mem":
132+
job.Memory = parseFloat(value)
133+
case "io":
134+
job.IO = parseFloat(value)
135+
case "iow":
136+
job.IOWait = parseFloat(value)
137+
case "maxvmem":
138+
job.MaxVMem = parseInt64(value)
139+
case "maxrss":
140+
job.MaxRSS = parseInt64(value)
141+
case "arid":
142+
job.ArID = value
143+
}
144+
}
145+
146+
if job.JobNumber != 0 {
147+
jobs = append(jobs, job)
148+
}
149+
150+
return jobs, nil
151+
}
152+
153+
// ParseQacctOutput parses the output of the qacct command and returns
154+
// a slice of JobDetail.
155+
func ParseQAcctJobOutput(output string) ([]JobDetail, error) {
156+
scanner := bufio.NewScanner(strings.NewReader(output))
157+
jobs, err := ParseQacctJobOutputWithScanner(scanner)
158+
if err != nil {
159+
return nil, err
160+
}
161+
return jobs, nil
162+
}
163+
164+
func parseInt(value string) int {
165+
i, _ := strconv.Atoi(value)
166+
return i
167+
}
168+
169+
func parseInt64(value string) int64 {
170+
i, _ := strconv.ParseInt(value, 10, 64)
171+
return i
172+
}
173+
174+
func parseFloat(value string) float64 {
175+
f, _ := strconv.ParseFloat(value, 64)
176+
return f
177+
}

0 commit comments

Comments
 (0)