diff --git a/pkg/qacct/parse.go b/pkg/qacct/parse.go deleted file mode 100644 index d14c8c5..0000000 --- a/pkg/qacct/parse.go +++ /dev/null @@ -1,177 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qacct - -import ( - "bufio" - "strconv" - "strings" -) - -func ParseQacctJobOutputWithScanner(scanner *bufio.Scanner) ([]JobDetail, error) { - var jobs []JobDetail - var job JobDetail - - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "==============================================================") { - if job.JobNumber != 0 { - jobs = append(jobs, job) - } - job = JobDetail{} - continue - } - - parts := strings.SplitN(line, " ", 2) - if len(parts) < 2 { - continue - } - - key := strings.TrimSpace(parts[0]) - value := strings.TrimSpace(parts[1]) - - switch key { - case "qname": - job.QName = value - case "hostname": - job.HostName = value - case "group": - job.Group = value - case "owner": - job.Owner = value - case "project": - job.Project = value - case "department": - job.Department = value - case "jobname": - job.JobName = value - case "jobnumber": - job.JobNumber = parseInt64(value) - case "taskid": - job.TaskID = parseInt64(value) - case "pe_taskid": - job.PETaskID = value - case "account": - job.Account = value - case "priority": - job.Priority = parseInt64(value) - case "qsub_time": - job.QSubTime = value - case "submit_cmd_line": - job.SubmitCommandLine = value - case "start_time": - job.StartTime = value - case "end_time": - job.EndTime = value - case "granted_pe": - job.GrantedPE = value - case "slots": - job.Slots = parseInt64(value) - case "failed": - job.Failed = parseInt64(value) - case "exit_status": - job.ExitStatus = parseInt64(value) - case "ru_wallclock": - job.RuWallClock = parseFloat(value) - case "ru_utime": - job.RuUTime = parseFloat(value) - case "ru_stime": - job.RuSTime = parseFloat(value) - case "ru_maxrss": - job.RuMaxRSS = parseInt64(value) - case "ru_ixrss": - job.RuIXRSS = parseInt64(value) - case "ru_ismrss": - job.RuISMRSS = parseInt64(value) - case "ru_idrss": - job.RuIDRSS = parseInt64(value) - case "ru_isrss": - job.RuISRss = parseInt64(value) - case "ru_minflt": - job.RuMinFlt = parseInt64(value) - case "ru_majflt": - job.RuMajFlt = parseInt64(value) - case "ru_nswap": - job.RuNSwap = parseInt64(value) - case "ru_inblock": - job.RuInBlock = parseInt64(value) - case "ru_oublock": - job.RuOuBlock = parseInt64(value) - case "ru_msgsnd": - job.RuMsgSend = parseInt64(value) - case "ru_msgrcv": - job.RuMsgRcv = parseInt64(value) - case "ru_nsignals": - job.RuNSignals = parseInt64(value) - case "ru_nvcsw": - job.RuNVCSw = parseInt64(value) - case "ru_nivcsw": - job.RuNiVCSw = parseInt64(value) - case "wallclock": - job.WallClock = parseFloat(value) - case "cpu": - job.CPU = parseFloat(value) - case "mem": - job.Memory = parseFloat(value) - case "io": - job.IO = parseFloat(value) - case "iow": - job.IOWait = parseFloat(value) - case "maxvmem": - job.MaxVMem = parseInt64(value) - case "maxrss": - job.MaxRSS = parseInt64(value) - case "arid": - job.ArID = value - } - } - - if job.JobNumber != 0 { - jobs = append(jobs, job) - } - - return jobs, nil -} - -// ParseQacctOutput parses the output of the qacct command and returns -// a slice of JobDetail. -func ParseQAcctJobOutput(output string) ([]JobDetail, error) { - scanner := bufio.NewScanner(strings.NewReader(output)) - jobs, err := ParseQacctJobOutputWithScanner(scanner) - if err != nil { - return nil, err - } - return jobs, nil -} - -func parseInt(value string) int { - i, _ := strconv.Atoi(value) - return i -} - -func parseInt64(value string) int64 { - i, _ := strconv.ParseInt(value, 10, 64) - return i -} - -func parseFloat(value string) float64 { - f, _ := strconv.ParseFloat(value, 64) - return f -} diff --git a/pkg/qacct/parse_test.go b/pkg/qacct/parse_test.go deleted file mode 100644 index 9a44604..0000000 --- a/pkg/qacct/parse_test.go +++ /dev/null @@ -1,191 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qacct_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/hpc-gridware/go-clusterscheduler/pkg/qacct" -) - -var _ = Describe("Parse", func() { - var sampleOutput string - - BeforeEach(func() { - sampleOutput = `============================================================== -qname all.q -hostname master -group root -owner root -project NONE -department defaultdepartment -jobname sleep -jobnumber 8 -taskid 97 -pe_taskid NONE -account sge -priority 0 -qsub_time 2024-09-27 07:41:44.421951 -submit_cmd_line qsub -b y -t 1-100:2 sleep 0 -start_time 2024-09-27 07:42:07.272221 -end_time 2024-09-27 07:42:08.801865 -granted_pe NONE -slots 1 -failed 0 -exit_status 0 -ru_wallclock 1 -ru_utime 0.492 -ru_stime 0.234 -ru_maxrss 10300 -ru_ixrss 0 -ru_ismrss 0 -ru_idrss 0 -ru_isrss 0 -ru_minflt 572 -ru_majflt 0 -ru_nswap 0 -ru_inblock 0 -ru_oublock 3 -ru_msgsnd 0 -ru_msgrcv 0 -ru_nsignals 0 -ru_nvcsw 471 -ru_nivcsw 568 -wallclock 3.487 -cpu 0.726 -mem 0.002 -io 0.000 -iow 0.000 -maxvmem 21045248 -maxrss 10547200 -arid undefined -============================================================== -qname all.q -hostname master -group root -owner root -project NONE -department defaultdepartment -jobname sleep -jobnumber 8 -taskid 99 -pe_taskid NONE -account sge -priority 0 -qsub_time 2024-09-27 07:41:44.421951 -submit_cmd_line qsub -b y -t 1-100:2 sleep 0 -start_time 2024-09-27 07:42:07.265733 -end_time 2024-09-27 07:42:08.796845 -granted_pe NONE -slots 1 -failed 0 -exit_status 0 -ru_wallclock 1 -ru_utime 0.487 -ru_stime 0.240 -ru_maxrss 10348 -ru_ixrss 0 -ru_ismrss 0 -ru_idrss 0 -ru_isrss 0 -ru_minflt 567 -ru_majflt 0 -ru_nswap 0 -ru_inblock 0 -ru_oublock 3 -ru_msgsnd 0 -ru_msgrcv 0 -ru_nsignals 0 -ru_nvcsw 434 -ru_nivcsw 519 -wallclock 3.464 -cpu 0.726 -mem 0.002 -io 0.000 -iow 0.000 -maxvmem 21045248 -maxrss 10596352 -arid undefined -Total System Usage - WALLCLOCK UTIME STIME CPU MEMORY IO IOW -================================================================================================================ - 72 25.861 12.399 38.259 0.284 0.000 0.000` - }) - - Describe("ParseQAcctOutput", func() { - It("should parse qacct output correctly", func() { - jobs, err := qacct.ParseQAcctJobOutput(sampleOutput) - - Expect(err).To(BeNil()) - Expect(jobs).To(HaveLen(2)) - - job1 := jobs[0] - Expect(job1.QName).To(Equal("all.q")) - Expect(job1.HostName).To(Equal("master")) - Expect(job1.Group).To(Equal("root")) - Expect(job1.Owner).To(Equal("root")) - Expect(job1.Project).To(Equal("NONE")) - Expect(job1.Department).To(Equal("defaultdepartment")) - Expect(job1.JobName).To(Equal("sleep")) - Expect(job1.JobNumber).To(Equal(int64(8))) - Expect(job1.TaskID).To(Equal(int64(97))) - Expect(job1.Account).To(Equal("sge")) - Expect(job1.QSubTime).To(Equal("2024-09-27 07:41:44.421951")) - Expect(job1.StartTime).To(Equal("2024-09-27 07:42:07.272221")) - Expect(job1.EndTime).To(Equal("2024-09-27 07:42:08.801865")) - Expect(job1.Failed).To(Equal(int64(0))) - Expect(job1.ExitStatus).To(Equal(int64(0))) - Expect(job1.RuWallClock).To(Equal(1.0)) - Expect(job1.RuUTime).To(Equal(0.492)) - Expect(job1.RuSTime).To(Equal(0.234)) - Expect(job1.RuMaxRSS).To(Equal(int64(10300))) - Expect(job1.MaxVMem).To(Equal(int64(21045248))) - Expect(job1.MaxRSS).To(Equal(int64(10547200))) - - job2 := jobs[1] - Expect(job2.QName).To(Equal("all.q")) - Expect(job2.HostName).To(Equal("master")) - Expect(job2.JobNumber).To(Equal(int64(8))) - Expect(job2.TaskID).To(Equal(int64(99))) - Expect(job2.QSubTime).To(Equal("2024-09-27 07:41:44.421951")) - Expect(job2.StartTime).To(Equal("2024-09-27 07:42:07.265733")) - Expect(job2.EndTime).To(Equal("2024-09-27 07:42:08.796845")) - }) - - It("should handle empty input", func() { - jobs, err := qacct.ParseQAcctJobOutput("") - Expect(err).To(BeNil()) - Expect(jobs).To(BeEmpty()) - }) - - It("should handle input with only total system usage", func() { - input := `Total System Usage - WALLCLOCK UTIME STIME CPU MEMORY IO IOW -================================================================================================================ - 72 25.861 12.399 38.259 0.284 0.000 0.000` - - jobs, err := qacct.ParseQAcctJobOutput(input) - Expect(err).To(BeNil()) - Expect(jobs).To(BeEmpty()) - }) - - }) -}) diff --git a/pkg/qacct/qacct_impl.go b/pkg/qacct/qacct_impl.go deleted file mode 100644 index 84fb0cb..0000000 --- a/pkg/qacct/qacct_impl.go +++ /dev/null @@ -1,157 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qacct - -import ( - "fmt" - "os/exec" -) - -type QAcctImpl struct { - config CommandLineQAcctConfig -} - -type CommandLineQAcctConfig struct { - Executable string - AccountingFile string - DryRun bool -} - -func NewCommandLineQAcct(config CommandLineQAcctConfig) (*QAcctImpl, error) { - if config.Executable != "" { - // check if executable is reachable - _, err := exec.LookPath(config.Executable) - if err != nil { - return nil, fmt.Errorf("executable not found: %w", err) - } - } else { - if !config.DryRun { - // check if qacct is in the PATH - _, err := exec.LookPath("qacct") - if err != nil { - return nil, fmt.Errorf("qacct not found in PATH") - } - } - } - return &QAcctImpl{config: config}, nil -} - -func (q *QAcctImpl) WithAlternativeAccountingFile(accountingFile string) error { - q.config.AccountingFile = accountingFile - return nil -} - -func (q *QAcctImpl) WithDefaultAccountingFile() { - q.config.AccountingFile = "" -} - -func (q *QAcctImpl) NativeSpecification(args []string) (string, error) { - if q.config.AccountingFile != "" { - args = append(args, "-f", q.config.AccountingFile) - } - - if q.config.DryRun { - return fmt.Sprintf("Dry run: %s %v", q.config.Executable, args), nil - } - - command := exec.Command(q.config.Executable, args...) - - out, err := command.Output() - if err != nil { - return "", fmt.Errorf("failed to get output of qacct: %w", err) - } - return string(out), nil -} - -func (q *QAcctImpl) ListAdvanceReservations(arID string) ([]ReservationUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) JobsAccountedTo(accountString string) (Usage, error) { - return Usage{}, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) JobsStartedAfter(beginTime string) (Usage, error) { - return Usage{}, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) JobsStartedBefore(endTime string) (Usage, error) { - return Usage{}, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) JobsStartedLastDays(days int) (Usage, error) { - return Usage{}, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListDepartment(department string) ([]DepartmentUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListGroup(groupIDOrName string) ([]GroupUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListHost(host string) ([]HostUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListJobs(jobIDOrNameOrPattern string) ([]JobDetail, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) RequestComplexAttributes(attributes string) ([]JobInfo, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListOwner(owner string) ([]OwnerUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListParallelEnvironment(peName string) ([]PeUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListProject(project string) ([]ProjectUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListQueue(queue string) ([]QueueUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListJobUsageBySlots(usedSlots int) ([]SlotsUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ListTasks(jobID, taskIDRange string) ([]TaskUsage, error) { - return nil, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ShowHelp() (string, error) { - return "", fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ShowTotalSystemUsage() (Usage, error) { - return Usage{}, fmt.Errorf("not implemented") -} - -func (q *QAcctImpl) ShowJobDetails(jobID int) (JobDetail, error) { - return JobDetail{}, fmt.Errorf("not implemented") -} diff --git a/pkg/qacct/qacct_impl_test.go b/pkg/qacct/qacct_impl_test.go deleted file mode 100644 index fbd576b..0000000 --- a/pkg/qacct/qacct_impl_test.go +++ /dev/null @@ -1,72 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qacct_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/hpc-gridware/go-clusterscheduler/pkg/qacct" -) - -var _ = Describe("QacctImpl", func() { - - // We need to test the native specification. - - Context("Native specification", func() { - - It("should return the native specification", func() { - - q, err := qacct.NewCommandLineQAcct( - qacct.CommandLineQAcctConfig{ - Executable: "qacct", - }) - Expect(err).NotTo(HaveOccurred()) - - result, err := q.NativeSpecification([]string{"-help"}) - Expect(err).NotTo(HaveOccurred()) - Expect(result). - To(ContainSubstring("usage: qacct [options]")) - }) - - It("should return an error if the command does not exist", func() { - q, err := qacct.NewCommandLineQAcct( - qacct.CommandLineQAcctConfig{ - Executable: "nonexistent-command", - }) - Expect(err).To(HaveOccurred()) - Expect(q).To(BeNil()) - }) - - It("should return an error if the qacct argument is invalid", func() { - q, err := qacct.NewCommandLineQAcct( - qacct.CommandLineQAcctConfig{ - Executable: "qacct", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(q).NotTo(BeNil()) - - result, err := q.NativeSpecification([]string{"-invalid-argument"}) - Expect(err).To(HaveOccurred()) - Expect(result).To(BeEmpty()) - }) - - }) -}) diff --git a/pkg/qacct/qacct_suite_test.go b/pkg/qacct/qacct_suite_test.go deleted file mode 100644 index a8c0317..0000000 --- a/pkg/qacct/qacct_suite_test.go +++ /dev/null @@ -1,32 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qacct_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestQacct(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Qacct Suite") -} diff --git a/pkg/qstat/qstat_impl_test.go b/pkg/qstat/qstat_impl_test.go deleted file mode 100644 index 5ca7412..0000000 --- a/pkg/qstat/qstat_impl_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qstat_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/hpc-gridware/go-clusterscheduler/pkg/qstat" -) - -var _ = Describe("QstatImpl", func() { - - Context("NativeSpecification", func() { - - It("should return the command line", func() { - q, err := qstat.NewCommandLineQstat( - qstat.CommandLineQStatConfig{ - DryRun: true, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(q.NativeSpecification([]string{"-j", "123"})). - To(Equal("Dry run: qstat [-j 123]")) - }) - - It("should return the help", func() { - q, err := qstat.NewCommandLineQstat(qstat.CommandLineQStatConfig{ - DryRun: false, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(q.NativeSpecification([]string{"-help"})). - To(ContainSubstring("usage: qstat [options]")) - }) - - It("should work without arguments", func() { - q, err := qstat.NewCommandLineQstat(qstat.CommandLineQStatConfig{}) - Expect(err).NotTo(HaveOccurred()) - _, err = q.NativeSpecification(nil) - Expect(err).NotTo(HaveOccurred()) - }) - - }) - -}) diff --git a/pkg/qstat/qstat_suite_test.go b/pkg/qstat/qstat_suite_test.go deleted file mode 100644 index 22eef5a..0000000 --- a/pkg/qstat/qstat_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package qstat_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestQstat(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Qstat Suite") -} diff --git a/pkg/qsub/qsub.go b/pkg/qsub/qsub.go deleted file mode 100644 index 39f2d15..0000000 --- a/pkg/qsub/qsub.go +++ /dev/null @@ -1,44 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qsub - -import "context" - -// Qsub is the interface for submitting jobs using qsub. -type Qsub interface { - - // Submit submits a job with the given options and returns the job ID or an error. - Submit(ctx context.Context, opts JobOptions) (int64, string, error) - - // SubmitWithNativeSpecification submits a job with the given options and - // returns the job ID or an error. - SubmitWithNativeSpecification(ctx context.Context, args []string) (string, error) - - // SubmitSimple submits a simple job script with minimal options. - SubmitSimple(ctx context.Context, command string, args ...string) (int64, string, error) - - // SubmitSimpleBinary submits a simple executable with minimal options. - SubmitSimpleBinary(ctx context.Context, command string, args ...string) (int64, string, error) - - // SubmitWithQueue submits a job to a specific queue. - SubmitWithQueue(ctx context.Context, queue string, opts JobOptions) (int64, string, error) - - // Other simplified methods can be added here. -} diff --git a/pkg/qsub/qsub_impl.go b/pkg/qsub/qsub_impl.go deleted file mode 100644 index 3ef563b..0000000 --- a/pkg/qsub/qsub_impl.go +++ /dev/null @@ -1,338 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qsub - -import ( - "context" - "errors" - "fmt" - "os/exec" - "strconv" - "strings" -) - -// qsubClient is a concrete implementation of the Qsub interface. -type qsubClient struct { - config CommandLineQSubConfig -} - -type CommandLineQSubConfig struct { - QsubPath string - DryRun bool -} - -// NewCommandLineQSub creates a new Qsub client. -// If qsubPath is empty, it defaults to "qsub". -func NewCommandLineQSub(config CommandLineQSubConfig) (Qsub, error) { - if config.QsubPath == "" { - config.QsubPath = "qsub" - } - if config.DryRun == false { - _, err := exec.LookPath(config.QsubPath) - if err != nil { - return nil, fmt.Errorf("executable not found: %w", err) - } - } - return &qsubClient{config: config}, nil -} - -// SubmitWithNativeSpecification submits a job with the given options and -// returns the job submission output with the job ID or an error. -func (c *qsubClient) SubmitWithNativeSpecification(ctx context.Context, args []string) (string, error) { - if len(args) == 0 { - return "", errors.New("no arguments provided") - } - if c.config.DryRun { - return fmt.Sprintf("Dry run: qsub %s", strings.Join(args, " ")), nil - } - // execute qsub with the given options - cmd := exec.CommandContext(ctx, c.config.QsubPath, args...) - output, err := cmd.CombinedOutput() - if err != nil { - return "", fmt.Errorf("qsub error: %v, output: %s", err, string(output)) - } - return string(output), nil -} - -// Submit constructs and executes the qsub command based on the provided -// JobOptions. It returns the jobID, the raw output of the job submission -// command, and an error if the submission command failed. -func (c *qsubClient) Submit(ctx context.Context, opts JobOptions) (int64, string, error) { - // default to terse output if not specified, to parse the job ID - // (if terse is not specified, qsub does not return the job ID, it - // returns the full output of the qsub command) - if opts.Terse == nil { - opts.Terse = new(bool) - *opts.Terse = true - } - - cmdArgs, err := buildQsubArgs(opts) - if err != nil { - return 0, "", err - } - - output, err := c.SubmitWithNativeSpecification(ctx, cmdArgs) - if err != nil { - return 0, output, err - } - - outputStr := strings.TrimSpace(string(output)) - if outputStr == "" { - return 0, "", errors.New("qsub did not return a job ID") - } - - // if terse is specified, qsub returns just the job ID, so we need to - // strip any trailing newlines - if *opts.Terse { - outputStr = strings.TrimRight(outputStr, "\n") - // jobid could be a number or like 7.1-100:2 - jobidstr := strings.Split(outputStr, ".")[0] - // parse the job ID as an int64 - jobIDInt, err := strconv.ParseInt(jobidstr, 10, 64) - if err != nil { - return 0, outputStr, fmt.Errorf("invalid job ID: %s", jobidstr) - } - return jobIDInt, outputStr, nil - } - - return 0, outputStr, nil -} - -// SubmitSimple submits a job with just the command (expected to be a -// script in the path of the execution host) and arguments. -func (c *qsubClient) SubmitSimple(ctx context.Context, command string, args ...string) (int64, string, error) { - opts := JobOptions{ - Command: command, - CommandArgs: args, - } - return c.Submit(ctx, opts) -} - -// SubmitSimpleBinary submits a simple executable with minimal options. -func (c *qsubClient) SubmitSimpleBinary(ctx context.Context, command string, args ...string) (int64, string, error) { - binary := true - - opts := JobOptions{ - Command: command, - CommandArgs: args, - Binary: &binary, - } - return c.Submit(ctx, opts) -} - -// SubmitWithQueue submits a job to a specific queue. -func (c *qsubClient) SubmitWithQueue(ctx context.Context, queue string, opts JobOptions) (int64, string, error) { - opts.Queue = append(opts.Queue, queue) - return c.Submit(ctx, opts) -} - -// buildQsubArgs constructs the qsub command-line arguments from JobOptions. -func buildQsubArgs(opts JobOptions) ([]string, error) { - var args []string - - // Helper function to add flags - addFlag := func(flag string, value string) { - if value != "" { - args = append(args, flag, value) - } else { - args = append(args, flag) - } - } - - // Time options - if opts.StartTime != nil { - addFlag("-a", *opts.StartTime) - } - if opts.Deadline != nil { - addFlag("-dl", *opts.Deadline) - } - if opts.AdvanceReservationID != nil { - addFlag("-ar", *opts.AdvanceReservationID) - } - - // Resource options - if opts.Account != nil { - addFlag("-A", *opts.Account) - } - if opts.Project != nil { - addFlag("-P", *opts.Project) - } - if opts.Priority != nil { - addFlag("-p", fmt.Sprintf("%d", *opts.Priority)) - } - if len(opts.ResourcesHardRequest) > 0 { - var res []string - for k, v := range opts.ResourcesHardRequest { - res = append(res, fmt.Sprintf("%s=%s", k, v)) - } - addFlag("-hard", "") - addFlag("-l", strings.Join(res, ",")) - } - if len(opts.ResourcesSoftRequest) > 0 { - var res []string - for k, v := range opts.ResourcesSoftRequest { - res = append(res, fmt.Sprintf("%s=%s", k, v)) - } - addFlag("-soft", "") - addFlag("-l", strings.Join(res, ",")) - } - if len(opts.Queue) > 0 { - addFlag("-q", strings.Join(opts.Queue, ",")) - } - if opts.Slots != nil { - addFlag("-pe", *opts.Slots) - } - - // Output/Input options - if len(opts.StdErr) > 0 { - addFlag("-e", strings.Join(opts.StdErr, ",")) - } - if len(opts.StdOut) > 0 { - addFlag("-o", strings.Join(opts.StdOut, ",")) - } - if len(opts.StdIn) > 0 { - addFlag("-i", strings.Join(opts.StdIn, ",")) - } - - // Execution options - if opts.Binary != nil { - if *opts.Binary { - addFlag("-b", "y") - } else { - addFlag("-b", "n") - } - } - if opts.WorkingDir != nil { - addFlag("-cwd", *opts.WorkingDir) - } - if opts.CommandPrefix != nil { - addFlag("-C", *opts.CommandPrefix) - } - if opts.Shell != nil { - if *opts.Shell { - addFlag("-shell", "y") - } else { - addFlag("-shell", "n") - } - } - if opts.CommandInterpreter != nil { - addFlag("-S", *opts.CommandInterpreter) - } - if opts.JobName != nil { - addFlag("-N", *opts.JobName) - } - if opts.JobArray != nil { - addFlag("-t", *opts.JobArray) - } - if opts.MaxRunningTasks != nil { - addFlag("-tc", fmt.Sprintf("%d", *opts.MaxRunningTasks)) - } - - // Notification options - if opts.MailOptions != nil { - addFlag("-m", *opts.MailOptions) - } - if len(opts.MailList) > 0 { - addFlag("-M", strings.Join(opts.MailList, ",")) - } - if opts.Notify != nil && *opts.Notify { - addFlag("-notify", "") - } - - // Dependency options - if len(opts.HoldJobIDs) > 0 { - addFlag("-hold_jid", strings.Join(opts.HoldJobIDs, ",")) - } - if len(opts.HoldArrayJobIDs) > 0 { - addFlag("-hold_jid_ad", strings.Join(opts.HoldArrayJobIDs, ",")) - } - - // Other options - if opts.Checkpoint != nil { - addFlag("-ckpt", *opts.Checkpoint) - } - if opts.CheckpointSelector != nil { - addFlag("-c", *opts.CheckpointSelector) - } - if opts.MergeStdOutErr != nil { - if *opts.MergeStdOutErr { - addFlag("-j", "y") - } else { - addFlag("-j", "n") - } - } - if opts.Verify != nil && *opts.Verify { - addFlag("-verify", "") - } - if opts.ExportAllEnv { - addFlag("-V", "") - } - if len(opts.ExportVariables) > 0 { - var envVars []string - for k, v := range opts.ExportVariables { - envVars = append(envVars, fmt.Sprintf("%s=%s", k, v)) - } - addFlag("-v", strings.Join(envVars, ",")) - } - if opts.Hold != nil && *opts.Hold { - addFlag("-h", "y") - } - - if opts.Synchronize != nil && *opts.Synchronize { - addFlag("-sync", "y") - } - if opts.ReservationDesired != nil { - if *opts.ReservationDesired { - addFlag("-R", "y") - } else { - addFlag("-R", "n") - } - } - if opts.Restartable != nil { - if *opts.Restartable { - addFlag("-r", "y") - } else { - addFlag("-r", "n") - } - } - if opts.Clear != nil && *opts.Clear { - addFlag("-clear", "") - } - if opts.Terse != nil && *opts.Terse { - addFlag("-terse", "") - } - if opts.PTTY != nil { - if *opts.PTTY { - addFlag("-pty", "y") - } else { - addFlag("-pty", "n") - } - } - - // Command and arguments - if opts.Command != "" { - args = append(args, opts.Command) - if len(opts.CommandArgs) > 0 { - args = append(args, opts.CommandArgs...) - } - } - - return args, nil -} diff --git a/pkg/qsub/qsub_impl_test.go b/pkg/qsub/qsub_impl_test.go deleted file mode 100644 index 799c87a..0000000 --- a/pkg/qsub/qsub_impl_test.go +++ /dev/null @@ -1,72 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qsub_test - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/hpc-gridware/go-clusterscheduler/pkg/qsub" -) - -var _ = Describe("QsubImpl", func() { - - Context("NativeSpecification", func() { - - It("should return the native qsub command line specification for the given options", func() { - ctx := context.Background() - qs, err := qsub.NewCommandLineQSub(qsub.CommandLineQSubConfig{}) - Expect(err).NotTo(HaveOccurred()) - - output, err := qs.SubmitWithNativeSpecification(ctx, - []string{"-help"}) - Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("usage: qsub")) - }) - - It("should return the job ID", func() { - ctx := context.Background() - qs, err := qsub.NewCommandLineQSub(qsub.CommandLineQSubConfig{}) - Expect(err).NotTo(HaveOccurred()) - - output, err := qs.SubmitWithNativeSpecification(ctx, - []string{"-b", "y", "sleep", "10"}) - Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Your job")) - - jobId, output, err := qs.SubmitSimpleBinary(ctx, "sleep", "0") - Expect(err).NotTo(HaveOccurred()) - Expect(jobId).To(BeNumerically(">", 0)) - Expect(output).NotTo(Equal("")) - - //submit again - jobId2, output, err := qs.SubmitSimpleBinary(ctx, "sleep", "0") - Expect(err).NotTo(HaveOccurred()) - Expect(jobId2).To(BeNumerically(">", 0)) - Expect(output).NotTo(Equal("")) - // jobId2 should be higher than jobId - Expect(jobId2).To(BeNumerically(">", jobId)) - }) - - }) - -}) diff --git a/pkg/qsub/qsub_suite_test.go b/pkg/qsub/qsub_suite_test.go deleted file mode 100644 index 28c87cc..0000000 --- a/pkg/qsub/qsub_suite_test.go +++ /dev/null @@ -1,32 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qsub_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestQsub(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Qsub Suite") -} diff --git a/pkg/qsub/types.go b/pkg/qsub/types.go deleted file mode 100644 index 3740e81..0000000 --- a/pkg/qsub/types.go +++ /dev/null @@ -1,86 +0,0 @@ -/*___INFO__MARK_BEGIN__*/ -/************************************************************************* -* Copyright 2024 HPC-Gridware GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -************************************************************************/ -/*___INFO__MARK_END__*/ - -package qsub - -type JobOptions struct { - // Time options - StartTime *string `flag:"-a"` - Deadline *string `flag:"-dl"` - - AdvanceReservationID *string `flag:"-ar"` - - // Resource options - Account *string `flag:"-A"` - Project *string `flag:"-P"` - Priority *int `flag:"-p"` - - ResourcesHardRequest map[string]string `flag:"-l_hard"` - ResourcesSoftRequest map[string]string `flag:"-l_soft"` - - Queue []string `flag:"-q"` - Slots *string `flag:"-pe"` - - // Output/Input options - StdErr []string `flag:"-e"` - StdOut []string `flag:"-o"` - StdIn []string `flag:"-i"` - - // Execution options - Binary *bool `flag:"-b"` - WorkingDir *string `flag:"-cwd,-wd"` - CommandPrefix *string `flag:"-C"` - Shell *bool `flag:"-shell"` - CommandInterpreter *string `flag:"-S"` - JobName *string `flag:"-N"` - JobArray *string `flag:"-t"` - MaxRunningTasks *int `flag:"-tc"` - - // Notification options - MailOptions *string `flag:"-m"` - MailList []string `flag:"-M"` - Notify *bool `flag:"-notify"` - MailAddresses []string `flag:"-M"` - EmailOnStart *bool // Custom field for email on start - EmailOnEnd *bool // Custom field for email on end - - // Dependency options - HoldJobIDs []string `flag:"-hold_jid"` - HoldArrayJobIDs []string `flag:"-hold_jid_ad"` - - // Other options - Checkpoint *string `flag:"-ckpt"` - CheckpointSelector *string `flag:"-c"` - MergeStdOutErr *bool `flag:"-j"` - UseCurrentDir *bool `flag:"-cwd"` - Verify *bool `flag:"-verify"` - ExportAllEnv bool `flag:"-V"` - ExportVariables map[string]string `flag:"-v"` - Hold *bool `flag:"-h"` - Synchronize *bool `flag:"-sync"` - ReservationDesired *bool `flag:"-R"` - Restartable *bool `flag:"-r"` - Clear *bool `flag:"-clear"` - Terse *bool `flag:"-terse"` - PTTY *bool `flag:"-pty"` - - // Add other fields as necessary - Command string // The command to execute - CommandArgs []string // Arguments for the command -}